Compare commits
4 Commits
fix-merge-
...
0e59cf05a5
Author | SHA1 | Date | |
---|---|---|---|
![]() |
0e59cf05a5 | ||
![]() |
588fd0207d | ||
![]() |
6dea158de9 | ||
![]() |
77528a3426 |
@@ -44,7 +44,7 @@ services:
|
||||
- ..:/usr/src/paperless/paperless-ngx:delegated
|
||||
- ../.devcontainer/vscode:/usr/src/paperless/paperless-ngx/.vscode:delegated # VSCode config files
|
||||
- virtualenv:/usr/src/paperless/paperless-ngx/.venv # Virtual environment persisted in volume
|
||||
- /usr/src/paperless/paperless-ngx/src/paperless/static/frontend # Static frontend files exist only in container
|
||||
- /usr/src/paperless/paperless-ngx/src/documents/static/frontend # Static frontend files exist only in container
|
||||
- /usr/src/paperless/paperless-ngx/src/.pytest_cache
|
||||
- /usr/src/paperless/paperless-ngx/.ruff_cache
|
||||
- /usr/src/paperless/paperless-ngx/htmlcov
|
||||
@@ -58,11 +58,11 @@ services:
|
||||
PAPERLESS_TIKA_ENABLED: 1
|
||||
PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
|
||||
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
|
||||
PAPERLESS_STATICDIR: ./src/paperless/static
|
||||
PAPERLESS_STATICDIR: ./src/documents/static
|
||||
PAPERLESS_DEBUG: true
|
||||
|
||||
# Overrides default command so things don't shut down after the process ends.
|
||||
command: /bin/sh -c "chown -R paperless:paperless /usr/src/paperless/paperless-ngx/src/paperless/static/frontend && chown -R paperless:paperless /usr/src/paperless/paperless-ngx/.ruff_cache && while sleep 1000; do :; done"
|
||||
command: /bin/sh -c "chown -R paperless:paperless /usr/src/paperless/paperless-ngx/src/documents/static/frontend && chown -R paperless:paperless /usr/src/paperless/paperless-ngx/.ruff_cache && while sleep 1000; do :; done"
|
||||
|
||||
gotenberg:
|
||||
image: docker.io/gotenberg/gotenberg:8.17
|
||||
|
6
.github/workflows/ci.yml
vendored
@@ -430,13 +430,13 @@ jobs:
|
||||
name: Export frontend artifact from docker
|
||||
run: |
|
||||
docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
|
||||
docker cp frontend-extract:/usr/src/paperless/src/paperless/static/frontend src/paperless/static/frontend/
|
||||
docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
|
||||
-
|
||||
name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: frontend-compiled
|
||||
path: src/paperless/static/frontend/
|
||||
path: src/documents/static/frontend/
|
||||
retention-days: 7
|
||||
|
||||
build-release:
|
||||
@@ -476,7 +476,7 @@ jobs:
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: frontend-compiled
|
||||
path: src/paperless/static/frontend/
|
||||
path: src/documents/static/frontend/
|
||||
-
|
||||
name: Download documentation artifact
|
||||
uses: actions/download-artifact@v4
|
||||
|
2
.gitignore
vendored
@@ -94,7 +94,7 @@ scripts/nuke
|
||||
/export/
|
||||
|
||||
# this is where the compiled frontend is moved to.
|
||||
/src/paperless/static/frontend/
|
||||
/src/documents/static/frontend/
|
||||
|
||||
# mac os
|
||||
.DS_Store
|
||||
|
@@ -234,7 +234,7 @@ RUN --mount=type=cache,target=${UV_CACHE_DIR},id=python-cache \
|
||||
COPY --chown=1000:1000 ./src ./
|
||||
|
||||
# copy frontend
|
||||
COPY --from=compile-frontend --chown=1000:1000 /src/src/paperless/static/frontend/ ./paperless/static/frontend/
|
||||
COPY --from=compile-frontend --chown=1000:1000 /src/src/documents/static/frontend/ ./documents/static/frontend/
|
||||
|
||||
# add users, setup scripts
|
||||
# Mount the compiled frontend to expected location
|
||||
|
@@ -1,179 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
# Source: https://github.com/sameersbn/docker-gitlab/
|
||||
map_uidgid() {
|
||||
local -r usermap_original_uid=$(id -u paperless)
|
||||
local -r usermap_original_gid=$(id -g paperless)
|
||||
local -r usermap_new_uid=${USERMAP_UID:-$usermap_original_uid}
|
||||
local -r usermap_new_gid=${USERMAP_GID:-${usermap_original_gid:-$usermap_new_uid}}
|
||||
if [[ ${usermap_new_uid} != "${usermap_original_uid}" || ${usermap_new_gid} != "${usermap_original_gid}" ]]; then
|
||||
echo "Mapping UID and GID for paperless:paperless to $usermap_new_uid:$usermap_new_gid"
|
||||
usermod --non-unique --uid "${usermap_new_uid}" paperless
|
||||
groupmod --non-unique --gid "${usermap_new_gid}" paperless
|
||||
fi
|
||||
}
|
||||
|
||||
map_folders() {
|
||||
# Export these so they can be used in docker-prepare.sh
|
||||
export DATA_DIR="${PAPERLESS_DATA_DIR:-/usr/src/paperless/data}"
|
||||
export MEDIA_ROOT_DIR="${PAPERLESS_MEDIA_ROOT:-/usr/src/paperless/media}"
|
||||
export CONSUME_DIR="${PAPERLESS_CONSUMPTION_DIR:-/usr/src/paperless/consume}"
|
||||
}
|
||||
|
||||
custom_container_init() {
|
||||
# Mostly borrowed from the LinuxServer.io base image
|
||||
# https://github.com/linuxserver/docker-baseimage-ubuntu/tree/bionic/root/etc/cont-init.d
|
||||
local -r custom_script_dir="/custom-cont-init.d"
|
||||
# Tamper checking.
|
||||
# Don't run files which are owned by anyone except root
|
||||
# Don't run files which are writeable by others
|
||||
if [ -d "${custom_script_dir}" ]; then
|
||||
if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 ! -user root)" ]; then
|
||||
echo "**** Potential tampering with custom scripts detected ****"
|
||||
echo "**** The folder '${custom_script_dir}' must be owned by root ****"
|
||||
return 0
|
||||
fi
|
||||
if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 -perm -o+w)" ]; then
|
||||
echo "**** The folder '${custom_script_dir}' or some of contents have write permissions for others, which is a security risk. ****"
|
||||
echo "**** Please review the permissions and their contents to make sure they are owned by root, and can only be modified by root. ****"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Make sure custom init directory has files in it
|
||||
if [ -n "$(/bin/ls --almost-all "${custom_script_dir}" 2>/dev/null)" ]; then
|
||||
echo "[custom-init] files found in ${custom_script_dir} executing"
|
||||
# Loop over files in the directory
|
||||
for SCRIPT in "${custom_script_dir}"/*; do
|
||||
NAME="$(basename "${SCRIPT}")"
|
||||
if [ -f "${SCRIPT}" ]; then
|
||||
echo "[custom-init] ${NAME}: executing..."
|
||||
/bin/bash "${SCRIPT}"
|
||||
echo "[custom-init] ${NAME}: exited $?"
|
||||
elif [ ! -f "${SCRIPT}" ]; then
|
||||
echo "[custom-init] ${NAME}: is not a file"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "[custom-init] no custom files found exiting..."
|
||||
fi
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
initialize() {
|
||||
|
||||
# Setup environment from secrets before anything else
|
||||
# Check for a version of this var with _FILE appended
|
||||
# and convert the contents to the env var value
|
||||
# Source it so export is persistent
|
||||
# shellcheck disable=SC1091
|
||||
source /sbin/env-from-file.sh
|
||||
|
||||
# Change the user and group IDs if needed
|
||||
map_uidgid
|
||||
|
||||
# Check for overrides of certain folders
|
||||
map_folders
|
||||
|
||||
local -r export_dir="/usr/src/paperless/export"
|
||||
|
||||
for dir in \
|
||||
"${export_dir}" \
|
||||
"${DATA_DIR}" "${DATA_DIR}/index" \
|
||||
"${MEDIA_ROOT_DIR}" "${MEDIA_ROOT_DIR}/documents" "${MEDIA_ROOT_DIR}/documents/originals" "${MEDIA_ROOT_DIR}/documents/thumbnails" \
|
||||
"${CONSUME_DIR}"; do
|
||||
if [[ ! -d "${dir}" ]]; then
|
||||
echo "Creating directory ${dir}"
|
||||
mkdir --parents --verbose "${dir}"
|
||||
fi
|
||||
done
|
||||
|
||||
local -r tmp_dir="${PAPERLESS_SCRATCH_DIR:=/tmp/paperless}"
|
||||
echo "Creating directory scratch directory ${tmp_dir}"
|
||||
mkdir --parents --verbose "${tmp_dir}"
|
||||
|
||||
set +e
|
||||
echo "Adjusting permissions of paperless files. This may take a while."
|
||||
chown -R paperless:paperless "${tmp_dir}"
|
||||
for dir in \
|
||||
"${export_dir}" \
|
||||
"${DATA_DIR}" \
|
||||
"${MEDIA_ROOT_DIR}" \
|
||||
"${CONSUME_DIR}"; do
|
||||
find "${dir}" -not \( -user paperless -and -group paperless \) -exec chown --changes paperless:paperless {} +
|
||||
done
|
||||
set -e
|
||||
|
||||
"${gosu_cmd[@]}" /sbin/docker-prepare.sh
|
||||
|
||||
# Leave this last thing
|
||||
custom_container_init
|
||||
|
||||
}
|
||||
|
||||
install_languages() {
|
||||
echo "Installing languages..."
|
||||
|
||||
read -ra langs <<<"$1"
|
||||
|
||||
# Check that it is not empty
|
||||
if [ ${#langs[@]} -eq 0 ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Build list of packages to install
|
||||
to_install=()
|
||||
for lang in "${langs[@]}"; do
|
||||
pkg="tesseract-ocr-$lang"
|
||||
|
||||
if dpkg --status "$pkg" &>/dev/null; then
|
||||
echo "Package $pkg already installed!"
|
||||
continue
|
||||
else
|
||||
to_install+=("$pkg")
|
||||
fi
|
||||
done
|
||||
|
||||
# Use apt only when we install packages
|
||||
if [ ${#to_install[@]} -gt 0 ]; then
|
||||
apt-get update
|
||||
|
||||
for pkg in "${to_install[@]}"; do
|
||||
|
||||
if ! apt-cache show "$pkg" &>/dev/null; then
|
||||
echo "Skipped $pkg: Package not found! :("
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Installing package $pkg..."
|
||||
if ! apt-get --assume-yes install "$pkg" &>/dev/null; then
|
||||
echo "Could not install $pkg"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Paperless-ngx docker container starting..."
|
||||
|
||||
gosu_cmd=(gosu paperless)
|
||||
if [ "$(id --user)" == "$(id --user paperless)" ]; then
|
||||
gosu_cmd=()
|
||||
fi
|
||||
|
||||
# Install additional languages if specified
|
||||
if [[ -n "$PAPERLESS_OCR_LANGUAGES" ]]; then
|
||||
install_languages "$PAPERLESS_OCR_LANGUAGES"
|
||||
fi
|
||||
|
||||
initialize
|
||||
|
||||
if [[ "$1" != "/"* ]]; then
|
||||
echo Executing management command "$@"
|
||||
exec "${gosu_cmd[@]}" python3 manage.py "$@"
|
||||
else
|
||||
echo Executing "$@"
|
||||
exec "$@"
|
||||
fi
|
@@ -390,7 +390,7 @@ Custom parsers can be added to Paperless-ngx to support more file types. In
|
||||
order to do that, you need to write the parser itself and announce its
|
||||
existence to Paperless-ngx.
|
||||
|
||||
The parser itself must extend `paperless.parsers.DocumentParser` and
|
||||
The parser itself must extend `documents.parsers.DocumentParser` and
|
||||
must implement the methods `parse` and `get_thumbnail`. You can provide
|
||||
your own implementation to `get_date` if you don't want to rely on
|
||||
Paperless-ngx' default date guessing mechanisms.
|
||||
@@ -418,7 +418,7 @@ class MyCustomParser(DocumentParser):
|
||||
```
|
||||
|
||||
If you encounter any issues during parsing, raise a
|
||||
`paperless.parsers.ParseError`.
|
||||
`documents.parsers.ParseError`.
|
||||
|
||||
The `self.tempdir` directory is a temporary directory that is guaranteed
|
||||
to be empty and removed after consumption finished. You can use that
|
||||
|
@@ -191,7 +191,7 @@ This might have multiple reasons.
|
||||
either manually or as part of the docker image build.
|
||||
|
||||
If the front end is still missing, make sure that the front end is
|
||||
compiled (files present in `src/paperless/static/frontend`). If it
|
||||
compiled (files present in `src/documents/static/frontend`). If it
|
||||
is not, you need to compile the front end yourself or download the
|
||||
release archive instead of cloning the repository.
|
||||
|
||||
|
@@ -16,7 +16,7 @@ classifiers = [
|
||||
|
||||
dependencies = [
|
||||
"bleach~=6.2.0",
|
||||
"celery[redis]~=5.4.0",
|
||||
"celery[redis]~=5.5.1",
|
||||
"channels~=4.2",
|
||||
"channels-redis~=4.2",
|
||||
"concurrent-log-handler~=0.9.25",
|
||||
@@ -200,60 +200,63 @@ lint.per-file-ignores."docker/wait-for-redis.py" = [
|
||||
"INP001",
|
||||
"T201",
|
||||
]
|
||||
lint.per-file-ignores."src/documents/file_handling.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/management/commands/document_consumer.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/management/commands/document_exporter.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/migrations/1012_fix_archive_files.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/models.py" = [
|
||||
"SIM115",
|
||||
]
|
||||
lint.per-file-ignores."src/documents/parsers.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/signals/handlers.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_consumer.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_file_handling.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_management.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_management_consumer.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_management_exporter.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_migration_archive_files.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_migration_document_pages_count.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_migration_mime_type.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/tests/test_sanity_check.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/documents/views.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/checks.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/file_handling.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/management/commands/document_consumer.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/management/commands/document_exporter.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/models.py" = [
|
||||
"SIM115",
|
||||
]
|
||||
lint.per-file-ignores."src/paperless/parsers.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/settings.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/signals/handlers.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_consumer.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_file_handling.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_management.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_management_consumer.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_management_exporter.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_migration_archive_files.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_migration_document_pages_count.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_migration_mime_type.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/tests/test_sanity_check.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
lint.per-file-ignores."src/paperless/views.py" = [
|
||||
"PTH",
|
||||
] # TODO Enable & remove
|
||||
|
@@ -100,7 +100,7 @@
|
||||
"with": "src/environments/environment.prod.ts"
|
||||
}
|
||||
],
|
||||
"outputPath": "../src/paperless/static/frontend/",
|
||||
"outputPath": "../src/documents/static/frontend/",
|
||||
"optimization": true,
|
||||
"outputHashing": "none",
|
||||
"sourceMap": false,
|
||||
|
@@ -1 +1,5 @@
|
||||
__all__ = []
|
||||
# this is here so that django finds the checks.
|
||||
from documents.checks import changed_password_check
|
||||
from documents.checks import parser_check
|
||||
|
||||
__all__ = ["changed_password_check", "parser_check"]
|
||||
|
214
src/documents/admin.py
Normal file
@@ -0,0 +1,214 @@
|
||||
from django.conf import settings
|
||||
from django.contrib import admin
|
||||
from guardian.admin import GuardedModelAdmin
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import Note
|
||||
from documents.models import PaperlessTask
|
||||
from documents.models import SavedView
|
||||
from documents.models import SavedViewFilterRule
|
||||
from documents.models import ShareLink
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
from auditlog.admin import LogEntryAdmin
|
||||
from auditlog.models import LogEntry
|
||||
|
||||
|
||||
class CorrespondentAdmin(GuardedModelAdmin):
|
||||
list_display = ("name", "match", "matching_algorithm")
|
||||
list_filter = ("matching_algorithm",)
|
||||
list_editable = ("match", "matching_algorithm")
|
||||
|
||||
|
||||
class TagAdmin(GuardedModelAdmin):
|
||||
list_display = ("name", "color", "match", "matching_algorithm")
|
||||
list_filter = ("matching_algorithm",)
|
||||
list_editable = ("color", "match", "matching_algorithm")
|
||||
search_fields = ("color", "name")
|
||||
|
||||
|
||||
class DocumentTypeAdmin(GuardedModelAdmin):
|
||||
list_display = ("name", "match", "matching_algorithm")
|
||||
list_filter = ("matching_algorithm",)
|
||||
list_editable = ("match", "matching_algorithm")
|
||||
|
||||
|
||||
class DocumentAdmin(GuardedModelAdmin):
|
||||
search_fields = ("correspondent__name", "title", "content", "tags__name")
|
||||
readonly_fields = (
|
||||
"added",
|
||||
"modified",
|
||||
"mime_type",
|
||||
"storage_type",
|
||||
"filename",
|
||||
"checksum",
|
||||
"archive_filename",
|
||||
"archive_checksum",
|
||||
"original_filename",
|
||||
"deleted_at",
|
||||
)
|
||||
|
||||
list_display_links = ("title",)
|
||||
|
||||
list_display = ("id", "title", "mime_type", "filename", "archive_filename")
|
||||
|
||||
list_filter = (
|
||||
("mime_type"),
|
||||
("archive_serial_number", admin.EmptyFieldListFilter),
|
||||
("archive_filename", admin.EmptyFieldListFilter),
|
||||
)
|
||||
|
||||
filter_horizontal = ("tags",)
|
||||
|
||||
ordering = ["-id"]
|
||||
|
||||
date_hierarchy = "created"
|
||||
|
||||
def has_add_permission(self, request):
|
||||
return False
|
||||
|
||||
def created_(self, obj):
|
||||
return obj.created.date().strftime("%Y-%m-%d")
|
||||
|
||||
created_.short_description = "Created"
|
||||
|
||||
def get_queryset(self, request): # pragma: no cover
|
||||
"""
|
||||
Include trashed documents
|
||||
"""
|
||||
return Document.global_objects.all()
|
||||
|
||||
def delete_queryset(self, request, queryset):
|
||||
from documents import index
|
||||
|
||||
with index.open_index_writer() as writer:
|
||||
for o in queryset:
|
||||
index.remove_document(writer, o)
|
||||
|
||||
super().delete_queryset(request, queryset)
|
||||
|
||||
def delete_model(self, request, obj):
|
||||
from documents import index
|
||||
|
||||
index.remove_document_from_index(obj)
|
||||
super().delete_model(request, obj)
|
||||
|
||||
def save_model(self, request, obj, form, change):
|
||||
from documents import index
|
||||
|
||||
index.add_or_update_document(obj)
|
||||
super().save_model(request, obj, form, change)
|
||||
|
||||
|
||||
class RuleInline(admin.TabularInline):
|
||||
model = SavedViewFilterRule
|
||||
|
||||
|
||||
class SavedViewAdmin(GuardedModelAdmin):
|
||||
list_display = ("name", "owner")
|
||||
|
||||
inlines = [RuleInline]
|
||||
|
||||
def get_queryset(self, request): # pragma: no cover
|
||||
return super().get_queryset(request).select_related("owner")
|
||||
|
||||
|
||||
class StoragePathInline(admin.TabularInline):
|
||||
model = StoragePath
|
||||
|
||||
|
||||
class StoragePathAdmin(GuardedModelAdmin):
|
||||
list_display = ("name", "path", "match", "matching_algorithm")
|
||||
list_filter = ("path", "matching_algorithm")
|
||||
list_editable = ("path", "match", "matching_algorithm")
|
||||
|
||||
|
||||
class TaskAdmin(admin.ModelAdmin):
|
||||
list_display = ("task_id", "task_file_name", "task_name", "date_done", "status")
|
||||
list_filter = ("status", "date_done", "task_name")
|
||||
search_fields = ("task_name", "task_id", "status", "task_file_name")
|
||||
readonly_fields = (
|
||||
"task_id",
|
||||
"task_file_name",
|
||||
"task_name",
|
||||
"status",
|
||||
"date_created",
|
||||
"date_started",
|
||||
"date_done",
|
||||
"result",
|
||||
)
|
||||
|
||||
|
||||
class NotesAdmin(GuardedModelAdmin):
|
||||
list_display = ("user", "created", "note", "document")
|
||||
list_filter = ("created", "user")
|
||||
list_display_links = ("created",)
|
||||
raw_id_fields = ("document",)
|
||||
search_fields = ("document__title",)
|
||||
|
||||
def get_queryset(self, request): # pragma: no cover
|
||||
return (
|
||||
super()
|
||||
.get_queryset(request)
|
||||
.select_related("user", "document__correspondent")
|
||||
)
|
||||
|
||||
|
||||
class ShareLinksAdmin(GuardedModelAdmin):
|
||||
list_display = ("created", "expiration", "document")
|
||||
list_filter = ("created", "expiration", "owner")
|
||||
list_display_links = ("created",)
|
||||
raw_id_fields = ("document",)
|
||||
|
||||
def get_queryset(self, request): # pragma: no cover
|
||||
return super().get_queryset(request).select_related("document__correspondent")
|
||||
|
||||
|
||||
class CustomFieldsAdmin(GuardedModelAdmin):
|
||||
fields = ("name", "created", "data_type")
|
||||
readonly_fields = ("created", "data_type")
|
||||
list_display = ("name", "created", "data_type")
|
||||
list_filter = ("created", "data_type")
|
||||
|
||||
|
||||
class CustomFieldInstancesAdmin(GuardedModelAdmin):
|
||||
fields = ("field", "document", "created", "value")
|
||||
readonly_fields = ("field", "document", "created", "value")
|
||||
list_display = ("field", "document", "value", "created")
|
||||
search_fields = ("document__title",)
|
||||
list_filter = ("created", "field")
|
||||
|
||||
def get_queryset(self, request): # pragma: no cover
|
||||
return (
|
||||
super()
|
||||
.get_queryset(request)
|
||||
.select_related("field", "document__correspondent")
|
||||
)
|
||||
|
||||
|
||||
admin.site.register(Correspondent, CorrespondentAdmin)
|
||||
admin.site.register(Tag, TagAdmin)
|
||||
admin.site.register(DocumentType, DocumentTypeAdmin)
|
||||
admin.site.register(Document, DocumentAdmin)
|
||||
admin.site.register(SavedView, SavedViewAdmin)
|
||||
admin.site.register(StoragePath, StoragePathAdmin)
|
||||
admin.site.register(PaperlessTask, TaskAdmin)
|
||||
admin.site.register(Note, NotesAdmin)
|
||||
admin.site.register(ShareLink, ShareLinksAdmin)
|
||||
admin.site.register(CustomField, CustomFieldsAdmin)
|
||||
admin.site.register(CustomFieldInstance, CustomFieldInstancesAdmin)
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
|
||||
class LogEntryAUDIT(LogEntryAdmin):
|
||||
def has_delete_permission(self, request, obj=None):
|
||||
return False
|
||||
|
||||
admin.site.unregister(LogEntry)
|
||||
admin.site.register(LogEntry, LogEntryAUDIT)
|
@@ -4,4 +4,30 @@ from django.utils.translation import gettext_lazy as _
|
||||
|
||||
class DocumentsConfig(AppConfig):
|
||||
name = "documents"
|
||||
verbose_name = _("Documents (legacy)")
|
||||
|
||||
verbose_name = _("Documents")
|
||||
|
||||
def ready(self):
|
||||
from documents.signals import document_consumption_finished
|
||||
from documents.signals import document_updated
|
||||
from documents.signals.handlers import add_inbox_tags
|
||||
from documents.signals.handlers import add_to_index
|
||||
from documents.signals.handlers import run_workflows_added
|
||||
from documents.signals.handlers import run_workflows_updated
|
||||
from documents.signals.handlers import set_correspondent
|
||||
from documents.signals.handlers import set_document_type
|
||||
from documents.signals.handlers import set_storage_path
|
||||
from documents.signals.handlers import set_tags
|
||||
|
||||
document_consumption_finished.connect(add_inbox_tags)
|
||||
document_consumption_finished.connect(set_correspondent)
|
||||
document_consumption_finished.connect(set_document_type)
|
||||
document_consumption_finished.connect(set_tags)
|
||||
document_consumption_finished.connect(set_storage_path)
|
||||
document_consumption_finished.connect(add_to_index)
|
||||
document_consumption_finished.connect(run_workflows_added)
|
||||
document_updated.connect(run_workflows_updated)
|
||||
|
||||
import documents.schema # noqa: F401
|
||||
|
||||
AppConfig.ready(self)
|
||||
|
@@ -13,15 +13,15 @@ from pikepdf import Page
|
||||
from pikepdf import PasswordError
|
||||
from pikepdf import Pdf
|
||||
|
||||
from paperless.converters import convert_from_tiff_to_pdf
|
||||
from paperless.data_models import ConsumableDocument
|
||||
from paperless.models import Tag
|
||||
from paperless.plugins.base import ConsumeTaskPlugin
|
||||
from paperless.plugins.base import StopConsumeTaskError
|
||||
from paperless.plugins.helpers import ProgressStatusOptions
|
||||
from paperless.utils import copy_basic_file_stats
|
||||
from paperless.utils import copy_file_with_basic_stats
|
||||
from paperless.utils import maybe_override_pixel_limit
|
||||
from documents.converters import convert_from_tiff_to_pdf
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.models import Tag
|
||||
from documents.plugins.base import ConsumeTaskPlugin
|
||||
from documents.plugins.base import StopConsumeTaskError
|
||||
from documents.plugins.helpers import ProgressStatusOptions
|
||||
from documents.utils import copy_basic_file_stats
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.utils import maybe_override_pixel_limit
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
@@ -123,7 +123,7 @@ class BarcodePlugin(ConsumeTaskPlugin):
|
||||
),
|
||||
).resolve()
|
||||
|
||||
from paperless import tasks
|
||||
from documents import tasks
|
||||
|
||||
# Create the split document tasks
|
||||
for new_document in self.separate_pages(separator_pages):
|
@@ -8,7 +8,7 @@ if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from zipfile import ZipFile
|
||||
|
||||
from paperless.models import Document
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
class BulkArchiveStrategy:
|
@@ -16,20 +16,20 @@ from django.conf import settings
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
|
||||
from paperless.data_models import ConsumableDocument
|
||||
from paperless.data_models import DocumentMetadataOverrides
|
||||
from paperless.data_models import DocumentSource
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import CustomField
|
||||
from paperless.models import CustomFieldInstance
|
||||
from paperless.models import Document
|
||||
from paperless.models import DocumentType
|
||||
from paperless.models import StoragePath
|
||||
from paperless.permissions import set_permissions_for_object
|
||||
from paperless.plugins.helpers import DocumentsStatusManager
|
||||
from paperless.tasks import bulk_update_documents
|
||||
from paperless.tasks import consume_file
|
||||
from paperless.tasks import update_document_content_maybe_archive_file
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentMetadataOverrides
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.permissions import set_permissions_for_object
|
||||
from documents.plugins.helpers import DocumentsStatusManager
|
||||
from documents.tasks import bulk_update_documents
|
||||
from documents.tasks import consume_file
|
||||
from documents.tasks import update_document_content_maybe_archive_file
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.contrib.auth.models import User
|
||||
@@ -179,6 +179,12 @@ def modify_custom_fields(
|
||||
custom_field.data_type
|
||||
]
|
||||
defaults[value_field] = value
|
||||
if (
|
||||
custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
|
||||
and doc_id in value
|
||||
):
|
||||
# Prevent self-linking
|
||||
continue
|
||||
CustomFieldInstance.objects.update_or_create(
|
||||
document_id=doc_id,
|
||||
field_id=field_id,
|
||||
@@ -220,7 +226,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
|
||||
try:
|
||||
Document.objects.filter(id__in=doc_ids).delete()
|
||||
|
||||
from paperless import index
|
||||
from documents import index
|
||||
|
||||
with index.open_index_writer() as writer:
|
||||
for id in doc_ids:
|
@@ -8,10 +8,10 @@ from typing import Final
|
||||
|
||||
from django.core.cache import cache
|
||||
|
||||
from paperless.models import Document
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from paperless.classifier import DocumentClassifier
|
||||
from documents.classifier import DocumentClassifier
|
||||
|
||||
logger = logging.getLogger("paperless.caching")
|
||||
|
||||
@@ -53,7 +53,7 @@ def get_suggestion_cache(document_id: int) -> SuggestionCacheData | None:
|
||||
The classifier needs to be matching in format and hash and the suggestions need to
|
||||
have been cached once.
|
||||
"""
|
||||
from paperless.classifier import DocumentClassifier
|
||||
from documents.classifier import DocumentClassifier
|
||||
|
||||
doc_key = get_suggestion_cache_key(document_id)
|
||||
cache_hits = cache.get_many([CLASSIFIER_VERSION_KEY, CLASSIFIER_HASH_KEY, doc_key])
|
88
src/documents/checks.py
Normal file
@@ -0,0 +1,88 @@
|
||||
import textwrap
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.checks import Error
|
||||
from django.core.checks import Warning
|
||||
from django.core.checks import register
|
||||
from django.core.exceptions import FieldError
|
||||
from django.db.utils import OperationalError
|
||||
from django.db.utils import ProgrammingError
|
||||
|
||||
from documents.signals import document_consumer_declaration
|
||||
from documents.templating.utils import convert_format_str_to_template_format
|
||||
|
||||
|
||||
@register()
|
||||
def changed_password_check(app_configs, **kwargs):
|
||||
from documents.models import Document
|
||||
from paperless.db import GnuPG
|
||||
|
||||
try:
|
||||
encrypted_doc = (
|
||||
Document.objects.filter(
|
||||
storage_type=Document.STORAGE_TYPE_GPG,
|
||||
)
|
||||
.only("pk", "storage_type")
|
||||
.first()
|
||||
)
|
||||
except (OperationalError, ProgrammingError, FieldError):
|
||||
return [] # No documents table yet
|
||||
|
||||
if encrypted_doc:
|
||||
if not settings.PASSPHRASE:
|
||||
return [
|
||||
Error(
|
||||
"The database contains encrypted documents but no password is set.",
|
||||
),
|
||||
]
|
||||
|
||||
if not GnuPG.decrypted(encrypted_doc.source_file):
|
||||
return [
|
||||
Error(
|
||||
textwrap.dedent(
|
||||
"""
|
||||
The current password doesn't match the password of the
|
||||
existing documents.
|
||||
|
||||
If you intend to change your password, you must first export
|
||||
all of the old documents, start fresh with the new password
|
||||
and then re-import them."
|
||||
""",
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
@register()
|
||||
def parser_check(app_configs, **kwargs):
|
||||
parsers = []
|
||||
for response in document_consumer_declaration.send(None):
|
||||
parsers.append(response[1])
|
||||
|
||||
if len(parsers) == 0:
|
||||
return [
|
||||
Error(
|
||||
"No parsers found. This is a bug. The consumer won't be "
|
||||
"able to consume any documents without parsers.",
|
||||
),
|
||||
]
|
||||
else:
|
||||
return []
|
||||
|
||||
|
||||
@register()
|
||||
def filename_format_check(app_configs, **kwargs):
|
||||
if settings.FILENAME_FORMAT:
|
||||
converted_format = convert_format_str_to_template_format(
|
||||
settings.FILENAME_FORMAT,
|
||||
)
|
||||
if converted_format != settings.FILENAME_FORMAT:
|
||||
return [
|
||||
Warning(
|
||||
f"Filename format {settings.FILENAME_FORMAT} is using the old style, please update to use double curly brackets",
|
||||
hint=converted_format,
|
||||
),
|
||||
]
|
||||
return []
|
@@ -17,12 +17,12 @@ if TYPE_CHECKING:
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
|
||||
from paperless.caching import CACHE_50_MINUTES
|
||||
from paperless.caching import CLASSIFIER_HASH_KEY
|
||||
from paperless.caching import CLASSIFIER_MODIFIED_KEY
|
||||
from paperless.caching import CLASSIFIER_VERSION_KEY
|
||||
from paperless.models import Document
|
||||
from paperless.models import MatchingModel
|
||||
from documents.caching import CACHE_50_MINUTES
|
||||
from documents.caching import CLASSIFIER_HASH_KEY
|
||||
from documents.caching import CLASSIFIER_MODIFIED_KEY
|
||||
from documents.caching import CLASSIFIER_VERSION_KEY
|
||||
from documents.models import Document
|
||||
from documents.models import MatchingModel
|
||||
|
||||
logger = logging.getLogger("paperless.classifier")
|
||||
|
@@ -4,14 +4,14 @@ from datetime import timezone
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
|
||||
from paperless.caching import CACHE_5_MINUTES
|
||||
from paperless.caching import CACHE_50_MINUTES
|
||||
from paperless.caching import CLASSIFIER_HASH_KEY
|
||||
from paperless.caching import CLASSIFIER_MODIFIED_KEY
|
||||
from paperless.caching import CLASSIFIER_VERSION_KEY
|
||||
from paperless.caching import get_thumbnail_modified_key
|
||||
from paperless.classifier import DocumentClassifier
|
||||
from paperless.models import Document
|
||||
from documents.caching import CACHE_5_MINUTES
|
||||
from documents.caching import CACHE_50_MINUTES
|
||||
from documents.caching import CLASSIFIER_HASH_KEY
|
||||
from documents.caching import CLASSIFIER_MODIFIED_KEY
|
||||
from documents.caching import CLASSIFIER_VERSION_KEY
|
||||
from documents.caching import get_thumbnail_modified_key
|
||||
from documents.classifier import DocumentClassifier
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
def suggestions_etag(request, pk: int) -> str | None:
|
@@ -15,38 +15,38 @@ from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
from rest_framework.reverse import reverse
|
||||
|
||||
from paperless.classifier import load_classifier
|
||||
from paperless.data_models import ConsumableDocument
|
||||
from paperless.data_models import DocumentMetadataOverrides
|
||||
from paperless.file_handling import create_source_path_directory
|
||||
from paperless.file_handling import generate_unique_filename
|
||||
from paperless.loggers import LoggingMixin
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import CustomField
|
||||
from paperless.models import CustomFieldInstance
|
||||
from paperless.models import Document
|
||||
from paperless.models import DocumentType
|
||||
from paperless.models import StoragePath
|
||||
from paperless.models import Tag
|
||||
from paperless.models import WorkflowTrigger
|
||||
from paperless.parsers import DocumentParser
|
||||
from paperless.parsers import ParseError
|
||||
from paperless.parsers import get_parser_class_for_mime_type
|
||||
from paperless.parsers import parse_date
|
||||
from paperless.permissions import set_permissions_for_object
|
||||
from paperless.plugins.base import AlwaysRunPluginMixin
|
||||
from paperless.plugins.base import ConsumeTaskPlugin
|
||||
from paperless.plugins.base import NoCleanupPluginMixin
|
||||
from paperless.plugins.base import NoSetupPluginMixin
|
||||
from paperless.plugins.helpers import ProgressManager
|
||||
from paperless.plugins.helpers import ProgressStatusOptions
|
||||
from paperless.signals import document_consumption_finished
|
||||
from paperless.signals import document_consumption_started
|
||||
from paperless.signals.handlers import run_workflows
|
||||
from paperless.templating.workflows import parse_w_workflow_placeholders
|
||||
from paperless.utils import copy_basic_file_stats
|
||||
from paperless.utils import copy_file_with_basic_stats
|
||||
from paperless.utils import run_subprocess
|
||||
from documents.classifier import load_classifier
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentMetadataOverrides
|
||||
from documents.file_handling import create_source_path_directory
|
||||
from documents.file_handling import generate_unique_filename
|
||||
from documents.loggers import LoggingMixin
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.parsers import DocumentParser
|
||||
from documents.parsers import ParseError
|
||||
from documents.parsers import get_parser_class_for_mime_type
|
||||
from documents.parsers import parse_date
|
||||
from documents.permissions import set_permissions_for_object
|
||||
from documents.plugins.base import AlwaysRunPluginMixin
|
||||
from documents.plugins.base import ConsumeTaskPlugin
|
||||
from documents.plugins.base import NoCleanupPluginMixin
|
||||
from documents.plugins.base import NoSetupPluginMixin
|
||||
from documents.plugins.helpers import ProgressManager
|
||||
from documents.plugins.helpers import ProgressStatusOptions
|
||||
from documents.signals import document_consumption_finished
|
||||
from documents.signals import document_consumption_started
|
||||
from documents.signals.handlers import run_workflows
|
||||
from documents.templating.workflows import parse_w_workflow_placeholders
|
||||
from documents.utils import copy_basic_file_stats
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.utils import run_subprocess
|
||||
from paperless_mail.parsers import MailDocumentParser
|
||||
|
||||
|
@@ -1,8 +1,8 @@
|
||||
from django.conf import settings as django_settings
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from documents.models import Document
|
||||
from paperless.config import GeneralConfig
|
||||
from paperless.models import Document
|
||||
|
||||
|
||||
def settings(request):
|
@@ -4,9 +4,9 @@ import img2pdf
|
||||
from django.conf import settings
|
||||
from PIL import Image
|
||||
|
||||
from paperless.utils import copy_basic_file_stats
|
||||
from paperless.utils import maybe_override_pixel_limit
|
||||
from paperless.utils import run_subprocess
|
||||
from documents.utils import copy_basic_file_stats
|
||||
from documents.utils import maybe_override_pixel_limit
|
||||
from documents.utils import run_subprocess
|
||||
|
||||
|
||||
def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
|
@@ -8,12 +8,12 @@ from typing import Final
|
||||
from django.conf import settings
|
||||
from pikepdf import Pdf
|
||||
|
||||
from paperless.consumer import ConsumerError
|
||||
from paperless.converters import convert_from_tiff_to_pdf
|
||||
from paperless.plugins.base import ConsumeTaskPlugin
|
||||
from paperless.plugins.base import NoCleanupPluginMixin
|
||||
from paperless.plugins.base import NoSetupPluginMixin
|
||||
from paperless.plugins.base import StopConsumeTaskError
|
||||
from documents.consumer import ConsumerError
|
||||
from documents.converters import convert_from_tiff_to_pdf
|
||||
from documents.plugins.base import ConsumeTaskPlugin
|
||||
from documents.plugins.base import NoCleanupPluginMixin
|
||||
from documents.plugins.base import NoSetupPluginMixin
|
||||
from documents.plugins.base import StopConsumeTaskError
|
||||
|
||||
logger = logging.getLogger("paperless.double_sided")
|
||||
|
@@ -2,9 +2,9 @@ import os
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from paperless.models import Document
|
||||
from paperless.templating.filepath import validate_filepath_template_and_render
|
||||
from paperless.templating.utils import convert_format_str_to_template_format
|
||||
from documents.models import Document
|
||||
from documents.templating.filepath import validate_filepath_template_and_render
|
||||
from documents.templating.utils import convert_format_str_to_template_format
|
||||
|
||||
|
||||
def create_source_path_directory(source_path):
|
950
src/documents/filters.py
Normal file
@@ -0,0 +1,950 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
import json
|
||||
import operator
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.db.models import Case
|
||||
from django.db.models import CharField
|
||||
from django.db.models import Count
|
||||
from django.db.models import Exists
|
||||
from django.db.models import IntegerField
|
||||
from django.db.models import OuterRef
|
||||
from django.db.models import Q
|
||||
from django.db.models import Subquery
|
||||
from django.db.models import Sum
|
||||
from django.db.models import Value
|
||||
from django.db.models import When
|
||||
from django.db.models.functions import Cast
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from django_filters.rest_framework import BooleanFilter
|
||||
from django_filters.rest_framework import Filter
|
||||
from django_filters.rest_framework import FilterSet
|
||||
from drf_spectacular.utils import extend_schema_field
|
||||
from guardian.utils import get_group_obj_perms_model
|
||||
from guardian.utils import get_user_obj_perms_model
|
||||
from rest_framework import serializers
|
||||
from rest_framework.filters import OrderingFilter
|
||||
from rest_framework_guardian.filters import ObjectPermissionsFilter
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import PaperlessTask
|
||||
from documents.models import ShareLink
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
CHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"]
|
||||
ID_KWARGS = ["in", "exact"]
|
||||
INT_KWARGS = ["exact", "gt", "gte", "lt", "lte", "isnull"]
|
||||
DATE_KWARGS = [
|
||||
"year",
|
||||
"month",
|
||||
"day",
|
||||
"date__gt",
|
||||
"date__gte",
|
||||
"gt",
|
||||
"gte",
|
||||
"date__lt",
|
||||
"date__lte",
|
||||
"lt",
|
||||
"lte",
|
||||
]
|
||||
|
||||
CUSTOM_FIELD_QUERY_MAX_DEPTH = 10
|
||||
CUSTOM_FIELD_QUERY_MAX_ATOMS = 20
|
||||
|
||||
|
||||
class CorrespondentFilterSet(FilterSet):
|
||||
class Meta:
|
||||
model = Correspondent
|
||||
fields = {
|
||||
"id": ID_KWARGS,
|
||||
"name": CHAR_KWARGS,
|
||||
}
|
||||
|
||||
|
||||
class TagFilterSet(FilterSet):
|
||||
class Meta:
|
||||
model = Tag
|
||||
fields = {
|
||||
"id": ID_KWARGS,
|
||||
"name": CHAR_KWARGS,
|
||||
}
|
||||
|
||||
|
||||
class DocumentTypeFilterSet(FilterSet):
|
||||
class Meta:
|
||||
model = DocumentType
|
||||
fields = {
|
||||
"id": ID_KWARGS,
|
||||
"name": CHAR_KWARGS,
|
||||
}
|
||||
|
||||
|
||||
class StoragePathFilterSet(FilterSet):
|
||||
class Meta:
|
||||
model = StoragePath
|
||||
fields = {
|
||||
"id": ID_KWARGS,
|
||||
"name": CHAR_KWARGS,
|
||||
"path": CHAR_KWARGS,
|
||||
}
|
||||
|
||||
|
||||
class ObjectFilter(Filter):
|
||||
def __init__(self, *, exclude=False, in_list=False, field_name=""):
|
||||
super().__init__()
|
||||
self.exclude = exclude
|
||||
self.in_list = in_list
|
||||
self.field_name = field_name
|
||||
|
||||
def filter(self, qs, value):
|
||||
if not value:
|
||||
return qs
|
||||
|
||||
try:
|
||||
object_ids = [int(x) for x in value.split(",")]
|
||||
except ValueError:
|
||||
return qs
|
||||
|
||||
if self.in_list:
|
||||
qs = qs.filter(**{f"{self.field_name}__id__in": object_ids}).distinct()
|
||||
else:
|
||||
for obj_id in object_ids:
|
||||
if self.exclude:
|
||||
qs = qs.exclude(**{f"{self.field_name}__id": obj_id})
|
||||
else:
|
||||
qs = qs.filter(**{f"{self.field_name}__id": obj_id})
|
||||
|
||||
return qs
|
||||
|
||||
|
||||
@extend_schema_field(serializers.BooleanField)
|
||||
class InboxFilter(Filter):
|
||||
def filter(self, qs, value):
|
||||
if value == "true":
|
||||
return qs.filter(tags__is_inbox_tag=True)
|
||||
elif value == "false":
|
||||
return qs.exclude(tags__is_inbox_tag=True)
|
||||
else:
|
||||
return qs
|
||||
|
||||
|
||||
@extend_schema_field(serializers.CharField)
|
||||
class TitleContentFilter(Filter):
|
||||
def filter(self, qs, value):
|
||||
if value:
|
||||
return qs.filter(Q(title__icontains=value) | Q(content__icontains=value))
|
||||
else:
|
||||
return qs
|
||||
|
||||
|
||||
@extend_schema_field(serializers.BooleanField)
|
||||
class SharedByUser(Filter):
|
||||
def filter(self, qs, value):
|
||||
ctype = ContentType.objects.get_for_model(self.model)
|
||||
UserObjectPermission = get_user_obj_perms_model()
|
||||
GroupObjectPermission = get_group_obj_perms_model()
|
||||
# see https://github.com/paperless-ngx/paperless-ngx/issues/5392, we limit subqueries
|
||||
# to 1 because Postgres doesn't like returning > 1 row, but all we care about is > 0
|
||||
return (
|
||||
qs.filter(
|
||||
owner_id=value,
|
||||
)
|
||||
.annotate(
|
||||
num_shared_users=Count(
|
||||
UserObjectPermission.objects.filter(
|
||||
content_type=ctype,
|
||||
object_pk=Cast(OuterRef("pk"), CharField()),
|
||||
).values("user_id")[:1],
|
||||
),
|
||||
)
|
||||
.annotate(
|
||||
num_shared_groups=Count(
|
||||
GroupObjectPermission.objects.filter(
|
||||
content_type=ctype,
|
||||
object_pk=Cast(OuterRef("pk"), CharField()),
|
||||
).values("group_id")[:1],
|
||||
),
|
||||
)
|
||||
.filter(
|
||||
Q(num_shared_users__gt=0) | Q(num_shared_groups__gt=0),
|
||||
)
|
||||
if value is not None
|
||||
else qs
|
||||
)
|
||||
|
||||
|
||||
class CustomFieldFilterSet(FilterSet):
|
||||
class Meta:
|
||||
model = CustomField
|
||||
fields = {
|
||||
"id": ID_KWARGS,
|
||||
"name": CHAR_KWARGS,
|
||||
}
|
||||
|
||||
|
||||
@extend_schema_field(serializers.CharField)
|
||||
class CustomFieldsFilter(Filter):
|
||||
def filter(self, qs, value):
|
||||
if value:
|
||||
fields_with_matching_selects = CustomField.objects.filter(
|
||||
extra_data__icontains=value,
|
||||
)
|
||||
option_ids = []
|
||||
if fields_with_matching_selects.count() > 0:
|
||||
for field in fields_with_matching_selects:
|
||||
options = field.extra_data.get("select_options", [])
|
||||
for _, option in enumerate(options):
|
||||
if option.get("label").lower().find(value.lower()) != -1:
|
||||
option_ids.extend([option.get("id")])
|
||||
return (
|
||||
qs.filter(custom_fields__field__name__icontains=value)
|
||||
| qs.filter(custom_fields__value_text__icontains=value)
|
||||
| qs.filter(custom_fields__value_bool__icontains=value)
|
||||
| qs.filter(custom_fields__value_int__icontains=value)
|
||||
| qs.filter(custom_fields__value_float__icontains=value)
|
||||
| qs.filter(custom_fields__value_date__icontains=value)
|
||||
| qs.filter(custom_fields__value_url__icontains=value)
|
||||
| qs.filter(custom_fields__value_monetary__icontains=value)
|
||||
| qs.filter(custom_fields__value_document_ids__icontains=value)
|
||||
| qs.filter(custom_fields__value_select__in=option_ids)
|
||||
)
|
||||
else:
|
||||
return qs
|
||||
|
||||
|
||||
class MimeTypeFilter(Filter):
|
||||
def filter(self, qs, value):
|
||||
if value:
|
||||
return qs.filter(mime_type__icontains=value)
|
||||
else:
|
||||
return qs
|
||||
|
||||
|
||||
class SelectField(serializers.CharField):
|
||||
def __init__(self, custom_field: CustomField):
|
||||
self._options = custom_field.extra_data["select_options"]
|
||||
super().__init__(max_length=16)
|
||||
|
||||
def to_internal_value(self, data):
|
||||
# If the supplied value is the option label instead of the ID
|
||||
try:
|
||||
data = next(
|
||||
option.get("id")
|
||||
for option in self._options
|
||||
if option.get("label") == data
|
||||
)
|
||||
except StopIteration:
|
||||
pass
|
||||
return super().to_internal_value(data)
|
||||
|
||||
|
||||
def handle_validation_prefix(func: Callable):
|
||||
"""
|
||||
Catch ValidationErrors raised by the wrapped function
|
||||
and add a prefix to the exception detail to track what causes the exception,
|
||||
similar to nested serializers.
|
||||
"""
|
||||
|
||||
def wrapper(*args, validation_prefix=None, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except serializers.ValidationError as e:
|
||||
raise serializers.ValidationError({validation_prefix: e.detail})
|
||||
|
||||
# Update the signature to include the validation_prefix argument
|
||||
old_sig = inspect.signature(func)
|
||||
new_param = inspect.Parameter("validation_prefix", inspect.Parameter.KEYWORD_ONLY)
|
||||
new_sig = old_sig.replace(parameters=[*old_sig.parameters.values(), new_param])
|
||||
|
||||
# Apply functools.wraps and manually set the new signature
|
||||
functools.update_wrapper(wrapper, func)
|
||||
wrapper.__signature__ = new_sig
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class CustomFieldQueryParser:
|
||||
EXPR_BY_CATEGORY = {
|
||||
"basic": ["exact", "in", "isnull", "exists"],
|
||||
"string": [
|
||||
"icontains",
|
||||
"istartswith",
|
||||
"iendswith",
|
||||
],
|
||||
"arithmetic": [
|
||||
"gt",
|
||||
"gte",
|
||||
"lt",
|
||||
"lte",
|
||||
"range",
|
||||
],
|
||||
"containment": ["contains"],
|
||||
}
|
||||
|
||||
SUPPORTED_EXPR_CATEGORIES = {
|
||||
CustomField.FieldDataType.STRING: ("basic", "string"),
|
||||
CustomField.FieldDataType.URL: ("basic", "string"),
|
||||
CustomField.FieldDataType.DATE: ("basic", "arithmetic"),
|
||||
CustomField.FieldDataType.BOOL: ("basic",),
|
||||
CustomField.FieldDataType.INT: ("basic", "arithmetic"),
|
||||
CustomField.FieldDataType.FLOAT: ("basic", "arithmetic"),
|
||||
CustomField.FieldDataType.MONETARY: ("basic", "string", "arithmetic"),
|
||||
CustomField.FieldDataType.DOCUMENTLINK: ("basic", "containment"),
|
||||
CustomField.FieldDataType.SELECT: ("basic",),
|
||||
}
|
||||
|
||||
DATE_COMPONENTS = [
|
||||
"year",
|
||||
"iso_year",
|
||||
"month",
|
||||
"day",
|
||||
"week",
|
||||
"week_day",
|
||||
"iso_week_day",
|
||||
"quarter",
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
validation_prefix,
|
||||
max_query_depth=10,
|
||||
max_atom_count=20,
|
||||
) -> None:
|
||||
"""
|
||||
A helper class that parses the query string into a `django.db.models.Q` for filtering
|
||||
documents based on custom field values.
|
||||
|
||||
The syntax of the query expression is illustrated with the below pseudo code rules:
|
||||
1. parse([`custom_field`, "exists", true]):
|
||||
matches documents with Q(custom_fields__field=`custom_field`)
|
||||
2. parse([`custom_field`, "exists", false]):
|
||||
matches documents with ~Q(custom_fields__field=`custom_field`)
|
||||
3. parse([`custom_field`, `op`, `value`]):
|
||||
matches documents with
|
||||
Q(custom_fields__field=`custom_field`, custom_fields__value_`type`__`op`= `value`)
|
||||
4. parse(["AND", [`q0`, `q1`, ..., `qn`]])
|
||||
-> parse(`q0`) & parse(`q1`) & ... & parse(`qn`)
|
||||
5. parse(["OR", [`q0`, `q1`, ..., `qn`]])
|
||||
-> parse(`q0`) | parse(`q1`) | ... | parse(`qn`)
|
||||
6. parse(["NOT", `q`])
|
||||
-> ~parse(`q`)
|
||||
|
||||
Args:
|
||||
validation_prefix: Used to generate the ValidationError message.
|
||||
max_query_depth: Limits the maximum nesting depth of queries.
|
||||
max_atom_count: Limits the maximum number of atoms (i.e., rule 1, 2, 3) in the query.
|
||||
|
||||
`max_query_depth` and `max_atom_count` can be set to guard against generating arbitrarily
|
||||
complex SQL queries.
|
||||
"""
|
||||
self._custom_fields: dict[int | str, CustomField] = {}
|
||||
self._validation_prefix = validation_prefix
|
||||
# Dummy ModelSerializer used to convert a Django models.Field to serializers.Field.
|
||||
self._model_serializer = serializers.ModelSerializer()
|
||||
# Used for sanity check
|
||||
self._max_query_depth = max_query_depth
|
||||
self._max_atom_count = max_atom_count
|
||||
self._current_depth = 0
|
||||
self._atom_count = 0
|
||||
# The set of annotations that we need to apply to the queryset
|
||||
self._annotations = {}
|
||||
|
||||
def parse(self, query: str) -> tuple[Q, dict[str, Count]]:
|
||||
"""
|
||||
Parses the query string into a `django.db.models.Q`
|
||||
and a set of annotations to be applied to the queryset.
|
||||
"""
|
||||
try:
|
||||
expr = json.loads(query)
|
||||
except json.JSONDecodeError:
|
||||
raise serializers.ValidationError(
|
||||
{self._validation_prefix: [_("Value must be valid JSON.")]},
|
||||
)
|
||||
return (
|
||||
self._parse_expr(expr, validation_prefix=self._validation_prefix),
|
||||
self._annotations,
|
||||
)
|
||||
|
||||
@handle_validation_prefix
|
||||
def _parse_expr(self, expr) -> Q:
|
||||
"""
|
||||
Applies rule (1, 2, 3) or (4, 5, 6) based on the length of the expr.
|
||||
"""
|
||||
with self._track_query_depth():
|
||||
if isinstance(expr, list | tuple):
|
||||
if len(expr) == 2:
|
||||
return self._parse_logical_expr(*expr)
|
||||
elif len(expr) == 3:
|
||||
return self._parse_atom(*expr)
|
||||
raise serializers.ValidationError(
|
||||
[_("Invalid custom field query expression")],
|
||||
)
|
||||
|
||||
@handle_validation_prefix
|
||||
def _parse_expr_list(self, exprs) -> list[Q]:
|
||||
"""
|
||||
Handles [`q0`, `q1`, ..., `qn`] in rule 4 & 5.
|
||||
"""
|
||||
if not isinstance(exprs, list | tuple) or not exprs:
|
||||
raise serializers.ValidationError(
|
||||
[_("Invalid expression list. Must be nonempty.")],
|
||||
)
|
||||
return [
|
||||
self._parse_expr(expr, validation_prefix=i) for i, expr in enumerate(exprs)
|
||||
]
|
||||
|
||||
def _parse_logical_expr(self, op, args) -> Q:
|
||||
"""
|
||||
Handles rule 4, 5, 6.
|
||||
"""
|
||||
op_lower = op.lower()
|
||||
|
||||
if op_lower == "not":
|
||||
return ~self._parse_expr(args, validation_prefix=1)
|
||||
|
||||
if op_lower == "and":
|
||||
op_func = operator.and_
|
||||
elif op_lower == "or":
|
||||
op_func = operator.or_
|
||||
else:
|
||||
raise serializers.ValidationError(
|
||||
{"0": [_("Invalid logical operator {op!r}").format(op=op)]},
|
||||
)
|
||||
|
||||
qs = self._parse_expr_list(args, validation_prefix="1")
|
||||
return functools.reduce(op_func, qs)
|
||||
|
||||
def _parse_atom(self, id_or_name, op, value) -> Q:
|
||||
"""
|
||||
Handles rule 1, 2, 3.
|
||||
"""
|
||||
# Guard against queries with too many conditions.
|
||||
self._atom_count += 1
|
||||
if self._atom_count > self._max_atom_count:
|
||||
raise serializers.ValidationError(
|
||||
[_("Maximum number of query conditions exceeded.")],
|
||||
)
|
||||
|
||||
custom_field = self._get_custom_field(id_or_name, validation_prefix="0")
|
||||
op = self._validate_atom_op(custom_field, op, validation_prefix="1")
|
||||
value = self._validate_atom_value(
|
||||
custom_field,
|
||||
op,
|
||||
value,
|
||||
validation_prefix="2",
|
||||
)
|
||||
|
||||
# Needed because not all DB backends support Array __contains
|
||||
if (
|
||||
custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
|
||||
and op == "contains"
|
||||
):
|
||||
return self._parse_atom_doc_link_contains(custom_field, value)
|
||||
|
||||
value_field_name = CustomFieldInstance.get_value_field_name(
|
||||
custom_field.data_type,
|
||||
)
|
||||
if (
|
||||
custom_field.data_type == CustomField.FieldDataType.MONETARY
|
||||
and op in self.EXPR_BY_CATEGORY["arithmetic"]
|
||||
):
|
||||
value_field_name = "value_monetary_amount"
|
||||
has_field = Q(custom_fields__field=custom_field)
|
||||
|
||||
# We need to use an annotation here because different atoms
|
||||
# might be referring to different instances of custom fields.
|
||||
annotation_name = f"_custom_field_filter_{len(self._annotations)}"
|
||||
|
||||
# Our special exists operator.
|
||||
if op == "exists":
|
||||
annotation = Count("custom_fields", filter=has_field)
|
||||
# A Document should have > 0 match if it has this field, or 0 if doesn't.
|
||||
query_op = "gt" if value else "exact"
|
||||
query = Q(**{f"{annotation_name}__{query_op}": 0})
|
||||
else:
|
||||
# Check if 1) custom field name matches, and 2) value satisfies condition
|
||||
field_filter = has_field & Q(
|
||||
**{f"custom_fields__{value_field_name}__{op}": value},
|
||||
)
|
||||
# Annotate how many matching custom fields each document has
|
||||
annotation = Count("custom_fields", filter=field_filter)
|
||||
# Filter document by count
|
||||
query = Q(**{f"{annotation_name}__gt": 0})
|
||||
|
||||
self._annotations[annotation_name] = annotation
|
||||
return query
|
||||
|
||||
@handle_validation_prefix
|
||||
def _get_custom_field(self, id_or_name):
|
||||
"""Get the CustomField instance by id or name."""
|
||||
if id_or_name in self._custom_fields:
|
||||
return self._custom_fields[id_or_name]
|
||||
|
||||
kwargs = (
|
||||
{"id": id_or_name} if isinstance(id_or_name, int) else {"name": id_or_name}
|
||||
)
|
||||
try:
|
||||
custom_field = CustomField.objects.get(**kwargs)
|
||||
except CustomField.DoesNotExist:
|
||||
raise serializers.ValidationError(
|
||||
[_("{name!r} is not a valid custom field.").format(name=id_or_name)],
|
||||
)
|
||||
self._custom_fields[custom_field.id] = custom_field
|
||||
self._custom_fields[custom_field.name] = custom_field
|
||||
return custom_field
|
||||
|
||||
@staticmethod
|
||||
def _split_op(full_op):
|
||||
*prefix, op = str(full_op).rsplit("__", maxsplit=1)
|
||||
prefix = prefix[0] if prefix else None
|
||||
return prefix, op
|
||||
|
||||
@handle_validation_prefix
|
||||
def _validate_atom_op(self, custom_field, raw_op):
|
||||
"""Check if the `op` is compatible with the type of the custom field."""
|
||||
prefix, op = self._split_op(raw_op)
|
||||
|
||||
# Check if the operator is supported for the current data_type.
|
||||
supported = False
|
||||
for category in self.SUPPORTED_EXPR_CATEGORIES[custom_field.data_type]:
|
||||
if op in self.EXPR_BY_CATEGORY[category]:
|
||||
supported = True
|
||||
break
|
||||
|
||||
# Check prefix
|
||||
if prefix is not None:
|
||||
if (
|
||||
prefix in self.DATE_COMPONENTS
|
||||
and custom_field.data_type == CustomField.FieldDataType.DATE
|
||||
):
|
||||
pass # ok - e.g., "year__exact" for date field
|
||||
else:
|
||||
supported = False # anything else is invalid
|
||||
|
||||
if not supported:
|
||||
raise serializers.ValidationError(
|
||||
[
|
||||
_("{data_type} does not support query expr {expr!r}.").format(
|
||||
data_type=custom_field.data_type,
|
||||
expr=raw_op,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
return raw_op
|
||||
|
||||
def _get_serializer_field(self, custom_field, full_op):
|
||||
"""Return a serializers.Field for value validation."""
|
||||
prefix, op = self._split_op(full_op)
|
||||
field = None
|
||||
|
||||
if op in ("isnull", "exists"):
|
||||
# `isnull` takes either True or False regardless of the data_type.
|
||||
field = serializers.BooleanField()
|
||||
elif (
|
||||
custom_field.data_type == CustomField.FieldDataType.DATE
|
||||
and prefix in self.DATE_COMPONENTS
|
||||
):
|
||||
# DateField admits queries in the form of `year__exact`, etc. These take integers.
|
||||
field = serializers.IntegerField()
|
||||
elif custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
|
||||
# We can be more specific here and make sure the value is a list.
|
||||
field = serializers.ListField(child=serializers.IntegerField())
|
||||
elif custom_field.data_type == CustomField.FieldDataType.SELECT:
|
||||
# We use this custom field to permit SELECT option names.
|
||||
field = SelectField(custom_field)
|
||||
elif custom_field.data_type == CustomField.FieldDataType.URL:
|
||||
# For URL fields we don't need to be strict about validation (e.g., for istartswith).
|
||||
field = serializers.CharField()
|
||||
else:
|
||||
# The general case: inferred from the corresponding field in CustomFieldInstance.
|
||||
value_field_name = CustomFieldInstance.get_value_field_name(
|
||||
custom_field.data_type,
|
||||
)
|
||||
model_field = CustomFieldInstance._meta.get_field(value_field_name)
|
||||
field_name = model_field.deconstruct()[0]
|
||||
field_class, field_kwargs = self._model_serializer.build_standard_field(
|
||||
field_name,
|
||||
model_field,
|
||||
)
|
||||
field = field_class(**field_kwargs)
|
||||
field.allow_null = False
|
||||
|
||||
# Need to set allow_blank manually because of the inconsistency in CustomFieldInstance validation.
|
||||
# See https://github.com/paperless-ngx/paperless-ngx/issues/7361.
|
||||
if isinstance(field, serializers.CharField):
|
||||
field.allow_blank = True
|
||||
|
||||
if op == "in":
|
||||
# `in` takes a list of values.
|
||||
field = serializers.ListField(child=field, allow_empty=False)
|
||||
elif op == "range":
|
||||
# `range` takes a list of values, i.e., [start, end].
|
||||
field = serializers.ListField(
|
||||
child=field,
|
||||
min_length=2,
|
||||
max_length=2,
|
||||
)
|
||||
|
||||
return field
|
||||
|
||||
@handle_validation_prefix
|
||||
def _validate_atom_value(self, custom_field, op, value):
|
||||
"""Check if `value` is valid for the custom field and `op`. Returns the validated value."""
|
||||
serializer_field = self._get_serializer_field(custom_field, op)
|
||||
return serializer_field.run_validation(value)
|
||||
|
||||
def _parse_atom_doc_link_contains(self, custom_field, value) -> Q:
|
||||
"""
|
||||
Handles document link `contains` in a way that is supported by all DB backends.
|
||||
"""
|
||||
|
||||
# If the value is an empty set,
|
||||
# this is trivially true for any document with not null document links.
|
||||
if not value:
|
||||
return Q(
|
||||
custom_fields__field=custom_field,
|
||||
custom_fields__value_document_ids__isnull=False,
|
||||
)
|
||||
|
||||
# First we look up reverse links from the requested documents.
|
||||
links = CustomFieldInstance.objects.filter(
|
||||
document_id__in=value,
|
||||
field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
|
||||
)
|
||||
|
||||
# Check if any of the requested IDs are missing.
|
||||
missing_ids = set(value) - set(link.document_id for link in links)
|
||||
if missing_ids:
|
||||
# The result should be an empty set in this case.
|
||||
return Q(id__in=[])
|
||||
|
||||
# Take the intersection of the reverse links - this should be what we are looking for.
|
||||
document_ids_we_want = functools.reduce(
|
||||
operator.and_,
|
||||
(set(link.value_document_ids) for link in links),
|
||||
)
|
||||
|
||||
return Q(id__in=document_ids_we_want)
|
||||
|
||||
@contextmanager
|
||||
def _track_query_depth(self):
|
||||
# guard against queries that are too deeply nested
|
||||
self._current_depth += 1
|
||||
if self._current_depth > self._max_query_depth:
|
||||
raise serializers.ValidationError([_("Maximum nesting depth exceeded.")])
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._current_depth -= 1
|
||||
|
||||
|
||||
@extend_schema_field(serializers.CharField)
|
||||
class CustomFieldQueryFilter(Filter):
|
||||
def __init__(self, validation_prefix):
|
||||
"""
|
||||
A filter that filters documents based on custom field name and value.
|
||||
|
||||
Args:
|
||||
validation_prefix: Used to generate the ValidationError message.
|
||||
"""
|
||||
super().__init__()
|
||||
self._validation_prefix = validation_prefix
|
||||
|
||||
def filter(self, qs, value):
|
||||
if not value:
|
||||
return qs
|
||||
|
||||
parser = CustomFieldQueryParser(
|
||||
self._validation_prefix,
|
||||
max_query_depth=CUSTOM_FIELD_QUERY_MAX_DEPTH,
|
||||
max_atom_count=CUSTOM_FIELD_QUERY_MAX_ATOMS,
|
||||
)
|
||||
q, annotations = parser.parse(value)
|
||||
|
||||
return qs.annotate(**annotations).filter(q)
|
||||
|
||||
|
||||
class DocumentFilterSet(FilterSet):
|
||||
is_tagged = BooleanFilter(
|
||||
label="Is tagged",
|
||||
field_name="tags",
|
||||
lookup_expr="isnull",
|
||||
exclude=True,
|
||||
)
|
||||
|
||||
tags__id__all = ObjectFilter(field_name="tags")
|
||||
|
||||
tags__id__none = ObjectFilter(field_name="tags", exclude=True)
|
||||
|
||||
tags__id__in = ObjectFilter(field_name="tags", in_list=True)
|
||||
|
||||
correspondent__id__none = ObjectFilter(field_name="correspondent", exclude=True)
|
||||
|
||||
document_type__id__none = ObjectFilter(field_name="document_type", exclude=True)
|
||||
|
||||
storage_path__id__none = ObjectFilter(field_name="storage_path", exclude=True)
|
||||
|
||||
is_in_inbox = InboxFilter()
|
||||
|
||||
title_content = TitleContentFilter()
|
||||
|
||||
owner__id__none = ObjectFilter(field_name="owner", exclude=True)
|
||||
|
||||
custom_fields__icontains = CustomFieldsFilter()
|
||||
|
||||
custom_fields__id__all = ObjectFilter(field_name="custom_fields__field")
|
||||
|
||||
custom_fields__id__none = ObjectFilter(
|
||||
field_name="custom_fields__field",
|
||||
exclude=True,
|
||||
)
|
||||
|
||||
custom_fields__id__in = ObjectFilter(
|
||||
field_name="custom_fields__field",
|
||||
in_list=True,
|
||||
)
|
||||
|
||||
has_custom_fields = BooleanFilter(
|
||||
label="Has custom field",
|
||||
field_name="custom_fields",
|
||||
lookup_expr="isnull",
|
||||
exclude=True,
|
||||
)
|
||||
|
||||
custom_field_query = CustomFieldQueryFilter("custom_field_query")
|
||||
|
||||
shared_by__id = SharedByUser()
|
||||
|
||||
mime_type = MimeTypeFilter()
|
||||
|
||||
class Meta:
|
||||
model = Document
|
||||
fields = {
|
||||
"id": ID_KWARGS,
|
||||
"title": CHAR_KWARGS,
|
||||
"content": CHAR_KWARGS,
|
||||
"archive_serial_number": INT_KWARGS,
|
||||
"created": DATE_KWARGS,
|
||||
"added": DATE_KWARGS,
|
||||
"modified": DATE_KWARGS,
|
||||
"original_filename": CHAR_KWARGS,
|
||||
"checksum": CHAR_KWARGS,
|
||||
"correspondent": ["isnull"],
|
||||
"correspondent__id": ID_KWARGS,
|
||||
"correspondent__name": CHAR_KWARGS,
|
||||
"tags__id": ID_KWARGS,
|
||||
"tags__name": CHAR_KWARGS,
|
||||
"document_type": ["isnull"],
|
||||
"document_type__id": ID_KWARGS,
|
||||
"document_type__name": CHAR_KWARGS,
|
||||
"storage_path": ["isnull"],
|
||||
"storage_path__id": ID_KWARGS,
|
||||
"storage_path__name": CHAR_KWARGS,
|
||||
"owner": ["isnull"],
|
||||
"owner__id": ID_KWARGS,
|
||||
"custom_fields": ["icontains"],
|
||||
}
|
||||
|
||||
|
||||
class ShareLinkFilterSet(FilterSet):
|
||||
class Meta:
|
||||
model = ShareLink
|
||||
fields = {
|
||||
"created": DATE_KWARGS,
|
||||
"expiration": DATE_KWARGS,
|
||||
}
|
||||
|
||||
|
||||
class PaperlessTaskFilterSet(FilterSet):
|
||||
acknowledged = BooleanFilter(
|
||||
label="Acknowledged",
|
||||
field_name="acknowledged",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = PaperlessTask
|
||||
fields = {
|
||||
"type": ["exact"],
|
||||
"task_name": ["exact"],
|
||||
"status": ["exact"],
|
||||
}
|
||||
|
||||
|
||||
class ObjectOwnedOrGrantedPermissionsFilter(ObjectPermissionsFilter):
|
||||
"""
|
||||
A filter backend that limits results to those where the requesting user
|
||||
has read object level permissions, owns the objects, or objects without
|
||||
an owner (for backwards compat)
|
||||
"""
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
objects_with_perms = super().filter_queryset(request, queryset, view)
|
||||
objects_owned = queryset.filter(owner=request.user)
|
||||
objects_unowned = queryset.filter(owner__isnull=True)
|
||||
return objects_with_perms | objects_owned | objects_unowned
|
||||
|
||||
|
||||
class ObjectOwnedPermissionsFilter(ObjectPermissionsFilter):
|
||||
"""
|
||||
A filter backend that limits results to those where the requesting user
|
||||
owns the objects or objects without an owner (for backwards compat)
|
||||
"""
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
if request.user.is_superuser:
|
||||
return queryset
|
||||
objects_owned = queryset.filter(owner=request.user)
|
||||
objects_unowned = queryset.filter(owner__isnull=True)
|
||||
return objects_owned | objects_unowned
|
||||
|
||||
|
||||
class DocumentsOrderingFilter(OrderingFilter):
|
||||
field_name = "ordering"
|
||||
prefix = "custom_field_"
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
param = request.query_params.get("ordering")
|
||||
if param and self.prefix in param:
|
||||
custom_field_id = int(param.split(self.prefix)[1])
|
||||
try:
|
||||
field = CustomField.objects.get(pk=custom_field_id)
|
||||
except CustomField.DoesNotExist:
|
||||
raise serializers.ValidationError(
|
||||
{self.prefix + str(custom_field_id): [_("Custom field not found")]},
|
||||
)
|
||||
|
||||
annotation = None
|
||||
match field.data_type:
|
||||
case CustomField.FieldDataType.STRING:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_text")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.INT:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_int")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.FLOAT:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_float")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.DATE:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_date")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.MONETARY:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_monetary_amount")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.SELECT:
|
||||
# Select options are a little more complicated since the value is the id of the option, not
|
||||
# the label. Additionally, to support sqlite we can't use StringAgg, so we need to create a
|
||||
# case statement for each option, setting the value to the index of the option in a list
|
||||
# sorted by label, and then summing the results to give a single value for the annotation
|
||||
|
||||
select_options = sorted(
|
||||
field.extra_data.get("select_options", []),
|
||||
key=lambda x: x.get("label"),
|
||||
)
|
||||
whens = [
|
||||
When(
|
||||
custom_fields__field_id=custom_field_id,
|
||||
custom_fields__value_select=option.get("id"),
|
||||
then=Value(idx, output_field=IntegerField()),
|
||||
)
|
||||
for idx, option in enumerate(select_options)
|
||||
]
|
||||
whens.append(
|
||||
When(
|
||||
custom_fields__field_id=custom_field_id,
|
||||
custom_fields__value_select__isnull=True,
|
||||
then=Value(
|
||||
len(select_options),
|
||||
output_field=IntegerField(),
|
||||
),
|
||||
),
|
||||
)
|
||||
annotation = Sum(
|
||||
Case(
|
||||
*whens,
|
||||
default=Value(0),
|
||||
output_field=IntegerField(),
|
||||
),
|
||||
)
|
||||
case CustomField.FieldDataType.DOCUMENTLINK:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_document_ids")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.URL:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_url")[:1],
|
||||
)
|
||||
case CustomField.FieldDataType.BOOL:
|
||||
annotation = Subquery(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
).values("value_bool")[:1],
|
||||
)
|
||||
|
||||
if not annotation:
|
||||
# Only happens if a new data type is added and not handled here
|
||||
raise ValueError("Invalid custom field data type")
|
||||
|
||||
queryset = (
|
||||
queryset.annotate(
|
||||
# We need to annotate the queryset with the custom field value
|
||||
custom_field_value=annotation,
|
||||
# We also need to annotate the queryset with a boolean for sorting whether the field exists
|
||||
has_field=Exists(
|
||||
CustomFieldInstance.objects.filter(
|
||||
document_id=OuterRef("id"),
|
||||
field_id=custom_field_id,
|
||||
),
|
||||
),
|
||||
)
|
||||
.order_by(
|
||||
"-has_field",
|
||||
param.replace(
|
||||
self.prefix + str(custom_field_id),
|
||||
"custom_field_value",
|
||||
),
|
||||
)
|
||||
.distinct()
|
||||
)
|
||||
|
||||
return super().filter_queryset(request, queryset, view)
|
@@ -38,10 +38,10 @@ from whoosh.scoring import TF_IDF
|
||||
from whoosh.util.times import timespan
|
||||
from whoosh.writing import AsyncWriter
|
||||
|
||||
from paperless.models import CustomFieldInstance
|
||||
from paperless.models import Document
|
||||
from paperless.models import Note
|
||||
from paperless.models import User
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import Note
|
||||
from documents.models import User
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.db.models import QuerySet
|
@@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand
|
||||
from django.db import connection
|
||||
from django.db import models
|
||||
|
||||
from paperless.models import Document
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
@@ -4,8 +4,8 @@ from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.core.management.base import CommandError
|
||||
|
||||
from documents.models import Document
|
||||
from paperless.db import GnuPG
|
||||
from paperless.models import Document
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
@@ -6,10 +6,10 @@ from django import db
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from paperless.management.commands.mixins import MultiProcessMixin
|
||||
from paperless.management.commands.mixins import ProgressBarMixin
|
||||
from paperless.models import Document
|
||||
from paperless.tasks import update_document_content_maybe_archive_file
|
||||
from documents.management.commands.mixins import MultiProcessMixin
|
||||
from documents.management.commands.mixins import ProgressBarMixin
|
||||
from documents.models import Document
|
||||
from documents.tasks import update_document_content_maybe_archive_file
|
||||
|
||||
logger = logging.getLogger("paperless.management.archiver")
|
||||
|
@@ -16,12 +16,12 @@ from django.core.management.base import CommandError
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
from watchdog.observers.polling import PollingObserver
|
||||
|
||||
from paperless.data_models import ConsumableDocument
|
||||
from paperless.data_models import DocumentMetadataOverrides
|
||||
from paperless.data_models import DocumentSource
|
||||
from paperless.models import Tag
|
||||
from paperless.parsers import is_file_ext_supported
|
||||
from paperless.tasks import consume_file
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentMetadataOverrides
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.models import Tag
|
||||
from documents.parsers import is_file_ext_supported
|
||||
from documents.tasks import consume_file
|
||||
|
||||
try:
|
||||
from inotifyrecursive import INotify
|
@@ -1,6 +1,6 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from paperless.tasks import train_classifier
|
||||
from documents.tasks import train_classifier
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
@@ -32,32 +32,32 @@ if TYPE_CHECKING:
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
from auditlog.models import LogEntry
|
||||
|
||||
from documents.file_handling import delete_empty_directories
|
||||
from documents.file_handling import generate_filename
|
||||
from documents.management.commands.mixins import CryptMixin
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import Note
|
||||
from documents.models import SavedView
|
||||
from documents.models import SavedViewFilterRule
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import UiSettings
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowAction
|
||||
from documents.models import WorkflowActionEmail
|
||||
from documents.models import WorkflowActionWebhook
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.settings import EXPORTER_ARCHIVE_NAME
|
||||
from documents.settings import EXPORTER_FILE_NAME
|
||||
from documents.settings import EXPORTER_THUMBNAIL_NAME
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from paperless import version
|
||||
from paperless.db import GnuPG
|
||||
from paperless.file_handling import delete_empty_directories
|
||||
from paperless.file_handling import generate_filename
|
||||
from paperless.management.commands.mixins import CryptMixin
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import CustomField
|
||||
from paperless.models import CustomFieldInstance
|
||||
from paperless.models import Document
|
||||
from paperless.models import DocumentType
|
||||
from paperless.models import Note
|
||||
from paperless.models import SavedView
|
||||
from paperless.models import SavedViewFilterRule
|
||||
from paperless.models import StoragePath
|
||||
from paperless.models import Tag
|
||||
from paperless.models import UiSettings
|
||||
from paperless.models import Workflow
|
||||
from paperless.models import WorkflowAction
|
||||
from paperless.models import WorkflowActionEmail
|
||||
from paperless.models import WorkflowActionWebhook
|
||||
from paperless.models import WorkflowTrigger
|
||||
from paperless.settings import EXPORTER_ARCHIVE_NAME
|
||||
from paperless.settings import EXPORTER_FILE_NAME
|
||||
from paperless.settings import EXPORTER_THUMBNAIL_NAME
|
||||
from paperless.utils import copy_file_with_basic_stats
|
||||
from paperless_mail.models import MailAccount
|
||||
from paperless_mail.models import MailRule
|
||||
|
@@ -7,9 +7,9 @@ import tqdm
|
||||
from django.core.management import BaseCommand
|
||||
from django.core.management import CommandError
|
||||
|
||||
from paperless.management.commands.mixins import MultiProcessMixin
|
||||
from paperless.management.commands.mixins import ProgressBarMixin
|
||||
from paperless.models import Document
|
||||
from documents.management.commands.mixins import MultiProcessMixin
|
||||
from documents.management.commands.mixins import ProgressBarMixin
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
@@ -21,24 +21,24 @@ from django.db.models.signals import m2m_changed
|
||||
from django.db.models.signals import post_save
|
||||
from filelock import FileLock
|
||||
|
||||
from documents.file_handling import create_source_path_directory
|
||||
from documents.management.commands.mixins import CryptMixin
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import Note
|
||||
from documents.models import Tag
|
||||
from documents.parsers import run_convert
|
||||
from documents.settings import EXPORTER_ARCHIVE_NAME
|
||||
from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME
|
||||
from documents.settings import EXPORTER_FILE_NAME
|
||||
from documents.settings import EXPORTER_THUMBNAIL_NAME
|
||||
from documents.signals.handlers import check_paths_and_prune_custom_fields
|
||||
from documents.signals.handlers import update_filename_and_move_files
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from paperless import version
|
||||
from paperless.file_handling import create_source_path_directory
|
||||
from paperless.management.commands.mixins import CryptMixin
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import CustomField
|
||||
from paperless.models import CustomFieldInstance
|
||||
from paperless.models import Document
|
||||
from paperless.models import DocumentType
|
||||
from paperless.models import Note
|
||||
from paperless.models import Tag
|
||||
from paperless.parsers import run_convert
|
||||
from paperless.settings import EXPORTER_ARCHIVE_NAME
|
||||
from paperless.settings import EXPORTER_CRYPTO_SETTINGS_NAME
|
||||
from paperless.settings import EXPORTER_FILE_NAME
|
||||
from paperless.settings import EXPORTER_THUMBNAIL_NAME
|
||||
from paperless.signals.handlers import check_paths_and_prune_custom_fields
|
||||
from paperless.signals.handlers import update_filename_and_move_files
|
||||
from paperless.utils import copy_file_with_basic_stats
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
from auditlog.registry import auditlog
|
@@ -1,9 +1,9 @@
|
||||
from django.core.management import BaseCommand
|
||||
from django.db import transaction
|
||||
|
||||
from paperless.management.commands.mixins import ProgressBarMixin
|
||||
from paperless.tasks import index_optimize
|
||||
from paperless.tasks import index_reindex
|
||||
from documents.management.commands.mixins import ProgressBarMixin
|
||||
from documents.tasks import index_optimize
|
||||
from documents.tasks import index_reindex
|
||||
|
||||
|
||||
class Command(ProgressBarMixin, BaseCommand):
|
@@ -4,8 +4,8 @@ import tqdm
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db.models.signals import post_save
|
||||
|
||||
from paperless.management.commands.mixins import ProgressBarMixin
|
||||
from paperless.models import Document
|
||||
from documents.management.commands.mixins import ProgressBarMixin
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
class Command(ProgressBarMixin, BaseCommand):
|
@@ -3,13 +3,13 @@ import logging
|
||||
import tqdm
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from paperless.classifier import load_classifier
|
||||
from paperless.management.commands.mixins import ProgressBarMixin
|
||||
from paperless.models import Document
|
||||
from paperless.signals.handlers import set_correspondent
|
||||
from paperless.signals.handlers import set_document_type
|
||||
from paperless.signals.handlers import set_storage_path
|
||||
from paperless.signals.handlers import set_tags
|
||||
from documents.classifier import load_classifier
|
||||
from documents.management.commands.mixins import ProgressBarMixin
|
||||
from documents.models import Document
|
||||
from documents.signals.handlers import set_correspondent
|
||||
from documents.signals.handlers import set_document_type
|
||||
from documents.signals.handlers import set_storage_path
|
||||
from documents.signals.handlers import set_tags
|
||||
|
||||
logger = logging.getLogger("paperless.management.retagger")
|
||||
|
@@ -1,7 +1,7 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from paperless.management.commands.mixins import ProgressBarMixin
|
||||
from paperless.sanity_checker import check_sanity
|
||||
from documents.management.commands.mixins import ProgressBarMixin
|
||||
from documents.sanity_checker import check_sanity
|
||||
|
||||
|
||||
class Command(ProgressBarMixin, BaseCommand):
|
@@ -6,10 +6,10 @@ import tqdm
|
||||
from django import db
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from paperless.management.commands.mixins import MultiProcessMixin
|
||||
from paperless.management.commands.mixins import ProgressBarMixin
|
||||
from paperless.models import Document
|
||||
from paperless.parsers import get_parser_class_for_mime_type
|
||||
from documents.management.commands.mixins import MultiProcessMixin
|
||||
from documents.management.commands.mixins import ProgressBarMixin
|
||||
from documents.models import Document
|
||||
from documents.parsers import get_parser_class_for_mime_type
|
||||
|
||||
|
||||
def _process_document(doc_id):
|
@@ -8,11 +8,11 @@ from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
|
||||
from django.core.management import CommandError
|
||||
|
||||
from paperless.settings import EXPORTER_CRYPTO_ALGO_NAME
|
||||
from paperless.settings import EXPORTER_CRYPTO_KEY_ITERATIONS_NAME
|
||||
from paperless.settings import EXPORTER_CRYPTO_KEY_SIZE_NAME
|
||||
from paperless.settings import EXPORTER_CRYPTO_SALT_NAME
|
||||
from paperless.settings import EXPORTER_CRYPTO_SETTINGS_NAME
|
||||
from documents.settings import EXPORTER_CRYPTO_ALGO_NAME
|
||||
from documents.settings import EXPORTER_CRYPTO_KEY_ITERATIONS_NAME
|
||||
from documents.settings import EXPORTER_CRYPTO_KEY_SIZE_NAME
|
||||
from documents.settings import EXPORTER_CRYPTO_SALT_NAME
|
||||
from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME
|
||||
|
||||
|
||||
class CryptFields(TypedDict):
|
@@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand
|
||||
from django.db import transaction
|
||||
from tqdm import tqdm
|
||||
|
||||
from paperless.management.commands.mixins import ProgressBarMixin
|
||||
from documents.management.commands.mixins import ProgressBarMixin
|
||||
|
||||
|
||||
class Command(BaseCommand, ProgressBarMixin):
|
@@ -5,20 +5,20 @@ import re
|
||||
from fnmatch import fnmatch
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from paperless.data_models import ConsumableDocument
|
||||
from paperless.data_models import DocumentSource
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import Document
|
||||
from paperless.models import DocumentType
|
||||
from paperless.models import MatchingModel
|
||||
from paperless.models import StoragePath
|
||||
from paperless.models import Tag
|
||||
from paperless.models import Workflow
|
||||
from paperless.models import WorkflowTrigger
|
||||
from paperless.permissions import get_objects_for_user_owner_aware
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import MatchingModel
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from paperless.classifier import DocumentClassifier
|
||||
from documents.classifier import DocumentClassifier
|
||||
|
||||
logger = logging.getLogger("paperless.matching")
|
||||
|
@@ -189,9 +189,9 @@ def parse_wrapper(parser, path, mime_type, file_name):
|
||||
|
||||
|
||||
def create_archive_version(doc, retry_count=3):
|
||||
from paperless.parsers import DocumentParser
|
||||
from paperless.parsers import ParseError
|
||||
from paperless.parsers import get_parser_class_for_mime_type
|
||||
from documents.parsers import DocumentParser
|
||||
from documents.parsers import ParseError
|
||||
from documents.parsers import get_parser_class_for_mime_type
|
||||
|
||||
logger.info(f"Regenerating archive document for document ID:{doc.id}")
|
||||
parser_class = get_parser_class_for_mime_type(doc.mime_type)
|
||||
@@ -271,7 +271,7 @@ def move_old_to_new_locations(apps, schema_editor):
|
||||
|
||||
# check that we can regenerate affected archive versions
|
||||
for doc_id in affected_document_ids:
|
||||
from paperless.parsers import get_parser_class_for_mime_type
|
||||
from documents.parsers import get_parser_class_for_mime_type
|
||||
|
||||
doc = Document.objects.get(id=doc_id)
|
||||
parser_class = get_parser_class_for_mime_type(doc.mime_type)
|
||||
|
@@ -9,7 +9,7 @@ from pathlib import Path
|
||||
from django.conf import settings
|
||||
from django.db import migrations
|
||||
|
||||
from paperless.parsers import run_convert
|
||||
from documents.parsers import run_convert
|
||||
|
||||
logger = logging.getLogger("paperless.migrations")
|
||||
|
||||
|
@@ -10,7 +10,7 @@ import gnupg
|
||||
from django.conf import settings
|
||||
from django.db import migrations
|
||||
|
||||
from paperless.parsers import run_convert
|
||||
from documents.parsers import run_convert
|
||||
|
||||
logger = logging.getLogger("paperless.migrations")
|
||||
|
||||
|
@@ -6,7 +6,7 @@ from django.db import models
|
||||
from django.db import transaction
|
||||
from filelock import FileLock
|
||||
|
||||
from paperless.templating.utils import convert_format_str_to_template_format
|
||||
from documents.templating.utils import convert_format_str_to_template_format
|
||||
|
||||
|
||||
def convert_from_format_to_template(apps, schema_editor):
|
||||
|
@@ -15,10 +15,10 @@ from typing import TYPE_CHECKING
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
from paperless.loggers import LoggingMixin
|
||||
from paperless.signals import document_consumer_declaration
|
||||
from paperless.utils import copy_file_with_basic_stats
|
||||
from paperless.utils import run_subprocess
|
||||
from documents.loggers import LoggingMixin
|
||||
from documents.signals import document_consumer_declaration
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.utils import run_subprocess
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import datetime
|
@@ -2,9 +2,9 @@ import abc
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from paperless.data_models import ConsumableDocument
|
||||
from paperless.data_models import DocumentMetadataOverrides
|
||||
from paperless.plugins.helpers import ProgressManager
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentMetadataOverrides
|
||||
from documents.plugins.helpers import ProgressManager
|
||||
|
||||
|
||||
class StopConsumeTaskError(Exception):
|
Before Width: | Height: | Size: 4.8 KiB After Width: | Height: | Size: 4.8 KiB |
@@ -10,8 +10,8 @@ from django.conf import settings
|
||||
from django.utils import timezone
|
||||
from tqdm import tqdm
|
||||
|
||||
from paperless.models import Document
|
||||
from paperless.models import PaperlessTask
|
||||
from documents.models import Document
|
||||
from documents.models import PaperlessTask
|
||||
|
||||
|
||||
class SanityCheckMessages:
|
2322
src/documents/serialisers.py
Normal file
11
src/documents/settings.py
Normal file
@@ -0,0 +1,11 @@
|
||||
# Defines the names of file/thumbnail for the manifest
|
||||
# for exporting/importing commands
|
||||
EXPORTER_FILE_NAME = "__exported_file_name__"
|
||||
EXPORTER_THUMBNAIL_NAME = "__exported_thumbnail_name__"
|
||||
EXPORTER_ARCHIVE_NAME = "__exported_archive_name__"
|
||||
|
||||
EXPORTER_CRYPTO_SETTINGS_NAME = "__crypto__"
|
||||
EXPORTER_CRYPTO_SALT_NAME = "__salt_hex__"
|
||||
EXPORTER_CRYPTO_KEY_ITERATIONS_NAME = "__key_iters__"
|
||||
EXPORTER_CRYPTO_KEY_SIZE_NAME = "__key_size__"
|
||||
EXPORTER_CRYPTO_ALGO_NAME = "__key_algo__"
|
@@ -23,35 +23,35 @@ from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
from guardian.shortcuts import remove_perm
|
||||
|
||||
from paperless import matching
|
||||
from paperless.caching import clear_document_caches
|
||||
from paperless.file_handling import create_source_path_directory
|
||||
from paperless.file_handling import delete_empty_directories
|
||||
from paperless.file_handling import generate_unique_filename
|
||||
from paperless.mail import send_email
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import CustomField
|
||||
from paperless.models import CustomFieldInstance
|
||||
from paperless.models import Document
|
||||
from paperless.models import DocumentType
|
||||
from paperless.models import MatchingModel
|
||||
from paperless.models import PaperlessTask
|
||||
from paperless.models import SavedView
|
||||
from paperless.models import Tag
|
||||
from paperless.models import Workflow
|
||||
from paperless.models import WorkflowAction
|
||||
from paperless.models import WorkflowRun
|
||||
from paperless.models import WorkflowTrigger
|
||||
from paperless.permissions import get_objects_for_user_owner_aware
|
||||
from paperless.permissions import set_permissions_for_object
|
||||
from paperless.templating.workflows import parse_w_workflow_placeholders
|
||||
from documents import matching
|
||||
from documents.caching import clear_document_caches
|
||||
from documents.file_handling import create_source_path_directory
|
||||
from documents.file_handling import delete_empty_directories
|
||||
from documents.file_handling import generate_unique_filename
|
||||
from documents.mail import send_email
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import MatchingModel
|
||||
from documents.models import PaperlessTask
|
||||
from documents.models import SavedView
|
||||
from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowAction
|
||||
from documents.models import WorkflowRun
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
from documents.permissions import set_permissions_for_object
|
||||
from documents.templating.workflows import parse_w_workflow_placeholders
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from paperless.classifier import DocumentClassifier
|
||||
from paperless.data_models import ConsumableDocument
|
||||
from paperless.data_models import DocumentMetadataOverrides
|
||||
from documents.classifier import DocumentClassifier
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentMetadataOverrides
|
||||
|
||||
logger = logging.getLogger("paperless.handlers")
|
||||
|
||||
@@ -578,7 +578,7 @@ def cleanup_custom_field_deletion(sender, instance: CustomField, **kwargs):
|
||||
|
||||
|
||||
def add_to_index(sender, document, **kwargs):
|
||||
from paperless import index
|
||||
from documents import index
|
||||
|
||||
index.add_or_update_document(document)
|
||||
|
||||
@@ -1271,7 +1271,7 @@ def before_task_publish_handler(sender=None, headers=None, body=None, **kwargs):
|
||||
https://docs.celeryq.dev/en/stable/internals/protocol.html#version-2
|
||||
|
||||
"""
|
||||
if "task" not in headers or headers["task"] != "paperless.tasks.consume_file":
|
||||
if "task" not in headers or headers["task"] != "documents.tasks.consume_file":
|
||||
# Assumption: this is only ever a v2 message
|
||||
return
|
||||
|
@@ -19,39 +19,39 @@ from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
from whoosh.writing import AsyncWriter
|
||||
|
||||
from paperless import index
|
||||
from paperless import sanity_checker
|
||||
from paperless.barcodes import BarcodePlugin
|
||||
from paperless.caching import clear_document_caches
|
||||
from paperless.classifier import DocumentClassifier
|
||||
from paperless.classifier import load_classifier
|
||||
from paperless.consumer import ConsumerPlugin
|
||||
from paperless.consumer import WorkflowTriggerPlugin
|
||||
from paperless.data_models import ConsumableDocument
|
||||
from paperless.data_models import DocumentMetadataOverrides
|
||||
from paperless.double_sided import CollatePlugin
|
||||
from paperless.file_handling import create_source_path_directory
|
||||
from paperless.file_handling import generate_unique_filename
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import CustomFieldInstance
|
||||
from paperless.models import Document
|
||||
from paperless.models import DocumentType
|
||||
from paperless.models import PaperlessTask
|
||||
from paperless.models import StoragePath
|
||||
from paperless.models import Tag
|
||||
from paperless.models import Workflow
|
||||
from paperless.models import WorkflowRun
|
||||
from paperless.models import WorkflowTrigger
|
||||
from paperless.parsers import DocumentParser
|
||||
from paperless.parsers import get_parser_class_for_mime_type
|
||||
from paperless.plugins.base import ConsumeTaskPlugin
|
||||
from paperless.plugins.base import ProgressManager
|
||||
from paperless.plugins.base import StopConsumeTaskError
|
||||
from paperless.plugins.helpers import ProgressStatusOptions
|
||||
from paperless.sanity_checker import SanityCheckFailedException
|
||||
from paperless.signals import document_updated
|
||||
from paperless.signals.handlers import cleanup_document_deletion
|
||||
from paperless.signals.handlers import run_workflows
|
||||
from documents import index
|
||||
from documents import sanity_checker
|
||||
from documents.barcodes import BarcodePlugin
|
||||
from documents.caching import clear_document_caches
|
||||
from documents.classifier import DocumentClassifier
|
||||
from documents.classifier import load_classifier
|
||||
from documents.consumer import ConsumerPlugin
|
||||
from documents.consumer import WorkflowTriggerPlugin
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentMetadataOverrides
|
||||
from documents.double_sided import CollatePlugin
|
||||
from documents.file_handling import create_source_path_directory
|
||||
from documents.file_handling import generate_unique_filename
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import PaperlessTask
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowRun
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.parsers import DocumentParser
|
||||
from documents.parsers import get_parser_class_for_mime_type
|
||||
from documents.plugins.base import ConsumeTaskPlugin
|
||||
from documents.plugins.base import ProgressManager
|
||||
from documents.plugins.base import StopConsumeTaskError
|
||||
from documents.plugins.helpers import ProgressStatusOptions
|
||||
from documents.sanity_checker import SanityCheckFailedException
|
||||
from documents.signals import document_updated
|
||||
from documents.signals.handlers import cleanup_document_deletion
|
||||
from documents.signals.handlers import run_workflows
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
from auditlog.models import LogEntry
|
Before Width: | Height: | Size: 679 B After Width: | Height: | Size: 679 B |
Before Width: | Height: | Size: 6.3 KiB After Width: | Height: | Size: 6.3 KiB |
0
src/documents/templating/__init__.py
Normal file
@@ -17,13 +17,13 @@ from jinja2 import make_logging_undefined
|
||||
from jinja2.sandbox import SandboxedEnvironment
|
||||
from jinja2.sandbox import SecurityError
|
||||
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import CustomField
|
||||
from paperless.models import CustomFieldInstance
|
||||
from paperless.models import Document
|
||||
from paperless.models import DocumentType
|
||||
from paperless.models import StoragePath
|
||||
from paperless.models import Tag
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
|
||||
logger = logging.getLogger("paperless.templating")
|
||||
|
0
src/documents/tests/__init__.py
Normal file
Before Width: | Height: | Size: 32 KiB After Width: | Height: | Size: 32 KiB |
@@ -1,8 +1,8 @@
|
||||
from factory import Faker
|
||||
from factory.django import DjangoModelFactory
|
||||
|
||||
from paperless.models import Correspondent
|
||||
from paperless.models import Document
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
class CorrespondentFactory(DjangoModelFactory):
|