paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-02 13:45:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev'

Browse Source
This commit is contained in:
shamoon 2023-02-16 20:07:50 -08:00
commit 38de2a7767
88 changed files with 9585 additions and 21104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.codecov.yml Normal file
View File

@ -0,0 +1,19 @@
# https://docs.codecov.com/docs/pull-request-comments
# codecov will only comment if coverage changes
comment:
require_changes: true
coverage:
status:
project:
default:
# https://docs.codecov.com/docs/commit-status#threshold
threshold: 1%
# https://docs.codecov.com/docs/commit-status#only_pulls
only_pulls: true
patch:
default:
# For the changed lines only, target 75% covered, but
# allow as low as 50%
target: 75%
threshold: 25%
only_pulls: true

172
.github/scripts/cleanup-tags.py vendored
View File

@ -7,6 +7,7 @@ import subprocess
from argparse import ArgumentParser
from typing import Dict
from typing import Final
from typing import Iterator
from typing import List
from typing import Optional
@ -15,16 +16,17 @@ from github import ContainerPackage
from github import GithubBranchApi
from github import GithubContainerRegistryApi
import docker
logger = logging.getLogger("cleanup-tags")
class DockerManifest2:
class ImageProperties:
"""
Data class wrapping the Docker Image Manifest Version 2.
Data class wrapping the properties of an entry in the image index
manifests list. It is NOT an actual image with layers, etc
See https://docs.docker.com/registry/spec/manifest-v2-2/
https://docs.docker.com/registry/spec/manifest-v2-2/
https://github.com/opencontainers/image-spec/blob/main/manifest.md
https://github.com/opencontainers/image-spec/blob/main/descriptor.md
"""
def __init__(self, data: Dict) -> None:
@ -41,6 +43,45 @@ class DockerManifest2:
self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
class ImageIndex:
"""
Data class wrapping up logic for an OCI Image Index
JSON data. Primary use is to access the manifests listing
See https://github.com/opencontainers/image-spec/blob/main/image-index.md
"""
def __init__(self, package_url: str, tag: str) -> None:
self.qualified_name = f"{package_url}:{tag}"
logger.info(f"Getting image index for {self.qualified_name}")
try:
proc = subprocess.run(
[
shutil.which("docker"),
"buildx",
"imagetools",
"inspect",
"--raw",
self.qualified_name,
],
capture_output=True,
check=True,
)
self._data = json.loads(proc.stdout)
except subprocess.CalledProcessError as e:
logger.error(
f"Failed to get image index for {self.qualified_name}: {e.stderr}",
)
raise e
@property
def image_pointers(self) -> Iterator[ImageProperties]:
for manifest_data in self._data["manifests"]:
yield ImageProperties(manifest_data)
class RegistryTagsCleaner:
"""
This is the base class for the image registry cleaning. Given a package
@ -87,7 +128,10 @@ class RegistryTagsCleaner:
def clean(self):
"""
This method will delete image versions, based on the selected tags to delete
This method will delete image versions, based on the selected tags to delete.
It behaves more like an unlinking than actual deletion. Removing the tag
simply removes a pointer to an image, but the actual image data remains accessible
if one has the sha256 digest of it.
"""
for tag_to_delete in self.tags_to_delete:
package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
@ -151,31 +195,17 @@ class RegistryTagsCleaner:
# Parse manifests to locate digests pointed to
for tag in sorted(self.tags_to_keep):
full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
logger.info(f"Checking manifest for {full_name}")
# TODO: It would be nice to use RegistryData from docker
# except the ID doesn't map to anything in the manifest
try:
proc = subprocess.run(
[
shutil.which("docker"),
"buildx",
"imagetools",
"inspect",
"--raw",
full_name,
],
capture_output=True,
image_index = ImageIndex(
f"ghcr.io/{self.repo_owner}/{self.package_name}",
tag,
)
manifest_list = json.loads(proc.stdout)
for manifest_data in manifest_list["manifests"]:
manifest = DockerManifest2(manifest_data)
for manifest in image_index.image_pointers:
if manifest.digest in untagged_versions:
logger.info(
f"Skipping deletion of {manifest.digest},"
f" referred to by {full_name}"
f" referred to by {image_index.qualified_name}"
f" for {manifest.platform}",
)
del untagged_versions[manifest.digest]
@ -247,64 +277,54 @@ class RegistryTagsCleaner:
# By default, keep anything which is tagged
self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
def check_tags_pull(self):
def check_remaining_tags_valid(self):
"""
This method uses the Docker Python SDK to confirm all tags which were
kept still pull, for all platforms.
Checks the non-deleted tags are still valid. The assumption is if the
manifest is can be inspected and each image manifest if points to can be
inspected, the image will still pull.
TODO: This is much slower (although more comprehensive). Maybe a Pool?
https://github.com/opencontainers/image-spec/blob/main/image-index.md
"""
logger.info("Beginning confirmation step")
client = docker.from_env()
imgs = []
a_tag_failed = False
for tag in sorted(self.tags_to_keep):
repository = f"ghcr.io/{self.repo_owner}/{self.package_name}"
for arch, variant in [("amd64", None), ("arm64", None), ("arm", "v7")]:
# From 11.2.0 onwards, qpdf is cross compiled, so there is a single arch, amd64
# skip others in this case
if "qpdf" in self.package_name and arch != "amd64" and tag == "11.2.0":
continue
# Skip beta and release candidate tags
elif "beta" in tag:
continue
# Build the platform name
if variant is not None:
platform = f"linux/{arch}/{variant}"
else:
platform = f"linux/{arch}"
try:
image_index = ImageIndex(
f"ghcr.io/{self.repo_owner}/{self.package_name}",
tag,
)
for manifest in image_index.image_pointers:
logger.info(f"Checking {manifest.digest} for {manifest.platform}")
try:
logger.info(f"Pulling {repository}:{tag} for {platform}")
image = client.images.pull(
repository=repository,
tag=tag,
platform=platform,
)
imgs.append(image)
except docker.errors.APIError as e:
logger.error(
f"Failed to pull {repository}:{tag}: {e}",
)
# This follows the pointer from the index to an actual image, layers and all
# Note the format is @
digest_name = f"ghcr.io/{self.repo_owner}/{self.package_name}@{manifest.digest}"
# Prevent out of space errors by removing after a few
# pulls
if len(imgs) > 50:
for image in imgs:
try:
client.images.remove(image.id)
except docker.errors.APIError as e:
err_str = str(e)
# Ignore attempts to remove images that are partly shared
# Ignore images which are somehow gone already
if (
"must be forced" not in err_str
and "No such image" not in err_str
):
logger.error(
f"Remove image ghcr.io/{self.repo_owner}/{self.package_name}:{tag} failed: {e}",
)
imgs = []
subprocess.run(
[
shutil.which("docker"),
"buildx",
"imagetools",
"inspect",
"--raw",
digest_name,
],
capture_output=True,
check=True,
)
except subprocess.CalledProcessError as e:
logger.error(f"Failed to inspect digest: {e.stderr}")
a_tag_failed = True
except subprocess.CalledProcessError as e:
a_tag_failed = True
logger.error(f"Failed to inspect: {e.stderr}")
continue
if a_tag_failed:
raise Exception("At least one image tag failed to inspect")
class MainImageTagsCleaner(RegistryTagsCleaner):
@ -366,7 +386,7 @@ class MainImageTagsCleaner(RegistryTagsCleaner):
class LibraryTagsCleaner(RegistryTagsCleaner):
"""
Exists for the off change that someday, the installer library images
Exists for the off chance that someday, the installer library images
will need their own logic
"""
@ -464,7 +484,7 @@ def _main():
# Verify remaining tags still pull
if args.is_manifest:
cleaner.check_tags_pull()
cleaner.check_remaining_tags_valid()
if __name__ == "__main__":

86
.github/workflows/ci.yml vendored
View File

@ -113,16 +113,12 @@ jobs:
PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
# Skip Tests which require convert
PAPERLESS_TEST_SKIP_CONVERT: 1
# Enable Gotenberg end to end testing
GOTENBERG_LIVE: 1
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
-
name: Start containers
run: |
@ -145,6 +141,10 @@ jobs:
run: |
sudo apt-get update -qq
sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
-
name: Configure ImageMagick
run: |
sudo cp docker/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
-
name: Install Python dependencies
run: |
@ -160,27 +160,14 @@ jobs:
cd src/
pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra
-
name: Get changed files
id: changed-files-specific
uses: tj-actions/changed-files@v35
name: Upload coverage to Codecov
if: ${{ matrix.python-version == env.DEFAULT_PYTHON_VERSION }}
uses: codecov/codecov-action@v3
with:
files: |
src/**
-
name: List all changed files
run: |
for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
echo "${file} was changed"
done
-
name: Publish coverage results
if: matrix.python-version == ${{ env.DEFAULT_PYTHON_VERSION }} && steps.changed-files-specific.outputs.any_changed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# https://github.com/coveralls-clients/coveralls-python/issues/251
run: |
cd src/
pipenv --python ${{ steps.setup-python.outputs.python-version }} run coveralls --service=github
# not required for public repos, but intermittently fails otherwise
token: ${{ secrets.CODECOV_TOKEN }}
# future expansion
flags: backend
-
name: Stop containers
if: always()
@ -347,7 +334,7 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build and push
uses: docker/build-push-action@v3
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
@ -442,21 +429,48 @@ jobs:
-
name: Move files
run: |
mkdir dist
mkdir dist/paperless-ngx
mkdir dist/paperless-ngx/scripts
cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/
cp paperless.conf.example dist/paperless-ngx/paperless.conf
cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
cp -r docker/ dist/paperless-ngx/docker
cp scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
cp -r src/ dist/paperless-ngx/src
cp -r docs/_build/html/ dist/paperless-ngx/docs
mv static dist/paperless-ngx
echo "Making dist folders"
for directory in dist \
dist/paperless-ngx \
dist/paperless-ngx/scripts;
do
mkdir --verbose --parents ${directory}
done
echo "Copying basic files"
for file_name in .dockerignore \
.env \
Dockerfile \
Pipfile \
Pipfile.lock \
requirements.txt \
LICENSE \
README.md \
paperless.conf.example \
gunicorn.conf.py
do
cp --verbose ${file_name} dist/paperless-ngx/
done
mv --verbose dist/paperless-ngx/paperless.conf.example paperless.conf
echo "Copying Docker related files"
cp --recursive docker/ dist/paperless-ngx/docker
echo "Copying startup scripts"
cp --verbose scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
echo "Copying source files"
cp --recursive src/ dist/paperless-ngx/src
echo "Copying documentation"
cp --recursive docs/_build/html/ dist/paperless-ngx/docs
mv --verbose static dist/paperless-ngx
-
name: Make release package
run: |
echo "Creating release archive"
cd dist
sudo chown -R 1000:1000 paperless-ngx/
tar -cJf paperless-ngx.tar.xz paperless-ngx/
-
name: Upload release artifact

2
.github/workflows/reusable-workflow-builder.yml vendored
View File

@ -45,7 +45,7 @@ jobs:
uses: docker/setup-qemu-action@v2
-
name: Build ${{ fromJSON(inputs.build-json).name }}
uses: docker/build-push-action@v3
uses: docker/build-push-action@v4
with:
context: .
file: ${{ inputs.dockerfile }}

2
.python-version
View File

@ -1 +1 @@
3.8.15
3.8.16

42
Dockerfile
View File

@ -1,12 +1,12 @@
# syntax=docker/dockerfile:1.4
# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md
# Stage: compile-frontend
# Purpose: Compiles the frontend
# Notes:
# - Does NPM stuff with Typescript and such
FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
# This stage compiles the frontend
# This stage runs once for the native platform, as the outputs are not
# dependent on target arch
# Inputs: None
COPY ./src-ui /src/src-ui
WORKDIR /src/src-ui
@ -16,15 +16,13 @@ RUN set -eux \
RUN set -eux \
&& ./node_modules/.bin/ng build --configuration production
# Stage: pipenv-base
# Purpose: Generates a requirements.txt file for building
# Comments:
# - pipenv dependencies are not left in the final image
# - pipenv can't touch the final image somehow
FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base
# This stage generates the requirements.txt file using pipenv
# This stage runs once for the native platform, as the outputs are not
# dependent on target arch
# This way, pipenv dependencies are not left in the final image
# nor can pipenv mess up the final image somehow
# Inputs: None
WORKDIR /usr/src/pipenv
COPY Pipfile* ./
@ -35,6 +33,10 @@ RUN set -eux \
&& echo "Generating requirement.txt" \
&& pipenv requirements > requirements.txt
# Stage: main-app
# Purpose: The final image
# Comments:
# - Don't leave anything extra in here
FROM python:3.9-slim-bullseye as main-app
LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
@ -61,10 +63,6 @@ ARG PSYCOPG2_VERSION
# Packages need for running
ARG RUNTIME_PACKAGES="\
# Python
python3 \
python3-pip \
python3-setuptools \
# General utils
curl \
# Docker specific
@ -128,7 +126,7 @@ RUN set -eux \
&& apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
&& rm -rf /var/lib/apt/lists/* \
&& echo "Installing supervisor" \
&& python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.4
&& python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.5
# Copy gunicorn config
# Changes very infrequently
@ -137,7 +135,6 @@ WORKDIR /usr/src/paperless/
COPY gunicorn.conf.py .
# setup docker-specific things
# Use mounts to avoid copying installer files into the image
# These change sometimes, but rarely
WORKDIR /usr/src/paperless/src/docker/
@ -179,7 +176,6 @@ RUN set -eux \
&& ./install_management_commands.sh
# Install the built packages from the installer library images
# Use mounts to avoid copying installer files into the image
# These change sometimes
RUN set -eux \
&& echo "Getting binaries" \
@ -203,7 +199,8 @@ RUN set -eux \
&& python3 -m pip list \
&& echo "Cleaning up image layer" \
&& cd ../ \
&& rm -rf paperless-ngx
&& rm -rf paperless-ngx \
&& rm paperless-ngx.tar.gz
WORKDIR /usr/src/paperless/src/
@ -247,11 +244,12 @@ COPY ./src ./
COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
# add users, setup scripts
# Mount the compiled frontend to expected location
RUN set -eux \
&& addgroup --gid 1000 paperless \
&& useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
&& chown -R paperless:paperless ../ \
&& gosu paperless python3 manage.py collectstatic --clear --no-input \
&& chown -R paperless:paperless /usr/src/paperless \
&& gosu paperless python3 manage.py collectstatic --clear --no-input --link \
&& gosu paperless python3 manage.py compilemessages
VOLUME ["/usr/src/paperless/data", \

6
Pipfile
View File

@ -12,6 +12,7 @@ name = "piwheels"
dateparser = "~=1.1"
django = "~=4.1"
django-cors-headers = "*"
django-compression-middleware = "*"
django-extensions = "*"
django-filter = "~=22.1"
djangorestframework = "~=3.14"
@ -53,21 +54,18 @@ nltk = "*"
pdf2image = "*"
flower = "*"
bleach = "*"
#
# Packages locked due to issues (try to check if these are fixed in a release every so often)
#
# Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
scipy = "==1.8.1"
# Newer versions aren't builting yet (see https://www.piwheels.org/project/cryptography/)
cryptography = "==38.0.1"
# Locked version until https://github.com/django/channels_redis/issues/332
# is resolved
channels-redis = "==3.4.1"
[dev-packages]
coveralls = "*"
factory-boy = "*"

154
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "5d6da0ede3fc7dd05c9a1d836bf8786285b10b6134e763d06ee90d6e1ccb2be7"
"sha256": "d70848276d3ac35fa361c15ac2d634344cdb08618790502669eee209fc16fa00"
},
"pipfile-spec": 6,
"requires": {},
@ -117,6 +117,93 @@
"index": "pypi",
"version": "==5.0.1"
},
"brotli": {
"hashes": [
"sha256:02177603aaca36e1fd21b091cb742bb3b305a569e2402f1ca38af471777fb019",
"sha256:11d3283d89af7033236fa4e73ec2cbe743d4f6a81d41bd234f24bf63dde979df",
"sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d",
"sha256:160c78292e98d21e73a4cc7f76a234390e516afcd982fa17e1422f7c6a9ce9c8",
"sha256:16d528a45c2e1909c2798f27f7bf0a3feec1dc9e50948e738b961618e38b6a7b",
"sha256:19598ecddd8a212aedb1ffa15763dd52a388518c4550e615aed88dc3753c0f0c",
"sha256:1c48472a6ba3b113452355b9af0a60da5c2ae60477f8feda8346f8fd48e3e87c",
"sha256:268fe94547ba25b58ebc724680609c8ee3e5a843202e9a381f6f9c5e8bdb5c70",
"sha256:269a5743a393c65db46a7bb982644c67ecba4b8d91b392403ad8a861ba6f495f",
"sha256:26d168aac4aaec9a4394221240e8a5436b5634adc3cd1cdf637f6645cecbf181",
"sha256:29d1d350178e5225397e28ea1b7aca3648fcbab546d20e7475805437bfb0a130",
"sha256:2aad0e0baa04517741c9bb5b07586c642302e5fb3e75319cb62087bd0995ab19",
"sha256:3148362937217b7072cf80a2dcc007f09bb5ecb96dae4617316638194113d5be",
"sha256:330e3f10cd01da535c70d09c4283ba2df5fb78e915bea0a28becad6e2ac010be",
"sha256:336b40348269f9b91268378de5ff44dc6fbaa2268194f85177b53463d313842a",
"sha256:3496fc835370da351d37cada4cf744039616a6db7d13c430035e901443a34daa",
"sha256:35a3edbe18e876e596553c4007a087f8bcfd538f19bc116917b3c7522fca0429",
"sha256:3b78a24b5fd13c03ee2b7b86290ed20efdc95da75a3557cc06811764d5ad1126",
"sha256:3b8b09a16a1950b9ef495a0f8b9d0a87599a9d1f179e2d4ac014b2ec831f87e7",
"sha256:3c1306004d49b84bd0c4f90457c6f57ad109f5cc6067a9664e12b7b79a9948ad",
"sha256:3ffaadcaeafe9d30a7e4e1e97ad727e4f5610b9fa2f7551998471e3736738679",
"sha256:40d15c79f42e0a2c72892bf407979febd9cf91f36f495ffb333d1d04cebb34e4",
"sha256:44bb8ff420c1d19d91d79d8c3574b8954288bdff0273bf788954064d260d7ab0",
"sha256:4688c1e42968ba52e57d8670ad2306fe92e0169c6f3af0089be75bbac0c64a3b",
"sha256:495ba7e49c2db22b046a53b469bbecea802efce200dffb69b93dd47397edc9b6",
"sha256:4d1b810aa0ed773f81dceda2cc7b403d01057458730e309856356d4ef4188438",
"sha256:503fa6af7da9f4b5780bb7e4cbe0c639b010f12be85d02c99452825dd0feef3f",
"sha256:56d027eace784738457437df7331965473f2c0da2c70e1a1f6fdbae5402e0389",
"sha256:5913a1177fc36e30fcf6dc868ce23b0453952c78c04c266d3149b3d39e1410d6",
"sha256:5b6ef7d9f9c38292df3690fe3e302b5b530999fa90014853dcd0d6902fb59f26",
"sha256:5bf37a08493232fbb0f8229f1824b366c2fc1d02d64e7e918af40acd15f3e337",
"sha256:5cb1e18167792d7d21e21365d7650b72d5081ed476123ff7b8cac7f45189c0c7",
"sha256:61a7ee1f13ab913897dac7da44a73c6d44d48a4adff42a5701e3239791c96e14",
"sha256:622a231b08899c864eb87e85f81c75e7b9ce05b001e59bbfbf43d4a71f5f32b2",
"sha256:68715970f16b6e92c574c30747c95cf8cf62804569647386ff032195dc89a430",
"sha256:6b2ae9f5f67f89aade1fab0f7fd8f2832501311c363a21579d02defa844d9296",
"sha256:6c772d6c0a79ac0f414a9f8947cc407e119b8598de7621f39cacadae3cf57d12",
"sha256:6d847b14f7ea89f6ad3c9e3901d1bc4835f6b390a9c71df999b0162d9bb1e20f",
"sha256:73fd30d4ce0ea48010564ccee1a26bfe39323fde05cb34b5863455629db61dc7",
"sha256:76ffebb907bec09ff511bb3acc077695e2c32bc2142819491579a695f77ffd4d",
"sha256:7bbff90b63328013e1e8cb50650ae0b9bac54ffb4be6104378490193cd60f85a",
"sha256:7cb81373984cc0e4682f31bc3d6be9026006d96eecd07ea49aafb06897746452",
"sha256:7ee83d3e3a024a9618e5be64648d6d11c37047ac48adff25f12fa4226cf23d1c",
"sha256:854c33dad5ba0fbd6ab69185fec8dab89e13cda6b7d191ba111987df74f38761",
"sha256:85f7912459c67eaab2fb854ed2bc1cc25772b300545fe7ed2dc03954da638649",
"sha256:87fdccbb6bb589095f413b1e05734ba492c962b4a45a13ff3408fa44ffe6479b",
"sha256:88c63a1b55f352b02c6ffd24b15ead9fc0e8bf781dbe070213039324922a2eea",
"sha256:8a674ac10e0a87b683f4fa2b6fa41090edfd686a6524bd8dedbd6138b309175c",
"sha256:8ed6a5b3d23ecc00ea02e1ed8e0ff9a08f4fc87a1f58a2530e71c0f48adf882f",
"sha256:93130612b837103e15ac3f9cbacb4613f9e348b58b3aad53721d92e57f96d46a",
"sha256:9744a863b489c79a73aba014df554b0e7a0fc44ef3f8a0ef2a52919c7d155031",
"sha256:9749a124280a0ada4187a6cfd1ffd35c350fb3af79c706589d98e088c5044267",
"sha256:97f715cf371b16ac88b8c19da00029804e20e25f30d80203417255d239f228b5",
"sha256:9bf919756d25e4114ace16a8ce91eb340eb57a08e2c6950c3cebcbe3dff2a5e7",
"sha256:9d12cf2851759b8de8ca5fde36a59c08210a97ffca0eb94c532ce7b17c6a3d1d",
"sha256:9ed4c92a0665002ff8ea852353aeb60d9141eb04109e88928026d3c8a9e5433c",
"sha256:a72661af47119a80d82fa583b554095308d6a4c356b2a554fdc2799bc19f2a43",
"sha256:afde17ae04d90fbe53afb628f7f2d4ca022797aa093e809de5c3cf276f61bbfa",
"sha256:b1375b5d17d6145c798661b67e4ae9d5496920d9265e2f00f1c2c0b5ae91fbde",
"sha256:b336c5e9cf03c7be40c47b5fd694c43c9f1358a80ba384a21969e0b4e66a9b17",
"sha256:b3523f51818e8f16599613edddb1ff924eeb4b53ab7e7197f85cbc321cdca32f",
"sha256:b43775532a5904bc938f9c15b77c613cb6ad6fb30990f3b0afaea82797a402d8",
"sha256:b663f1e02de5d0573610756398e44c130add0eb9a3fc912a09665332942a2efb",
"sha256:b83bb06a0192cccf1eb8d0a28672a1b79c74c3a8a5f2619625aeb6f28b3a82bb",
"sha256:ba72d37e2a924717990f4d7482e8ac88e2ef43fb95491eb6e0d124d77d2a150d",
"sha256:c2415d9d082152460f2bd4e382a1e85aed233abc92db5a3880da2257dc7daf7b",
"sha256:c83aa123d56f2e060644427a882a36b3c12db93727ad7a7b9efd7d7f3e9cc2c4",
"sha256:c8e521a0ce7cf690ca84b8cc2272ddaf9d8a50294fd086da67e517439614c755",
"sha256:cab1b5964b39607a66adbba01f1c12df2e55ac36c81ec6ed44f2fca44178bf1a",
"sha256:cb02ed34557afde2d2da68194d12f5719ee96cfb2eacc886352cb73e3808fc5d",
"sha256:cc0283a406774f465fb45ec7efb66857c09ffefbe49ec20b7882eff6d3c86d3a",
"sha256:cfc391f4429ee0a9370aa93d812a52e1fee0f37a81861f4fdd1f4fb28e8547c3",
"sha256:db844eb158a87ccab83e868a762ea8024ae27337fc7ddcbfcddd157f841fdfe7",
"sha256:defed7ea5f218a9f2336301e6fd379f55c655bea65ba2476346340a0ce6f74a1",
"sha256:e16eb9541f3dd1a3e92b89005e37b1257b157b7256df0e36bd7b33b50be73bcb",
"sha256:e1abbeef02962596548382e393f56e4c94acd286bd0c5afba756cffc33670e8a",
"sha256:e23281b9a08ec338469268f98f194658abfb13658ee98e2b7f85ee9dd06caa91",
"sha256:e2d9e1cbc1b25e22000328702b014227737756f4b5bf5c485ac1d8091ada078b",
"sha256:e48f4234f2469ed012a98f4b7874e7f7e173c167bed4934912a29e03167cf6b1",
"sha256:e4c4e92c14a57c9bd4cb4be678c25369bf7a092d55fd0866f759e425b9660806",
"sha256:ec1947eabbaf8e0531e8e899fc1d9876c179fc518989461f5d24e2223395a9e3",
"sha256:f909bbbc433048b499cb9db9e713b5d8d949e8c109a2a548502fb9aa8630f0b1"
],
"version": "==1.0.9"
},
"celery": {
"extras": [
"redis"
@ -353,6 +440,14 @@
"index": "pypi",
"version": "==2.4.0"
},
"django-compression-middleware": {
"hashes": [
"sha256:5e089b299a603f73254a495bf60d82eb8a7d4d636ba23dbc0ee4e4e911eee37d",
"sha256:71d4bcd09546cf88783150ac6467eb4168599534774b09b806be3017139c295b"
],
"index": "pypi",
"version": "==0.4.2"
},
"django-cors-headers": {
"hashes": [
"sha256:37e42883b5f1f2295df6b4bba96eb2417a14a03270cb24b2a07f021cd4487cf4",
@ -2003,6 +2098,63 @@
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==5.5.2"
},
"zstandard": {
"hashes": [
"sha256:04c298d381a3b6274b0a8001f0da0ec7819d052ad9c3b0863fe8c7f154061f76",
"sha256:0fde1c56ec118940974e726c2a27e5b54e71e16c6f81d0b4722112b91d2d9009",
"sha256:126aa8433773efad0871f624339c7984a9c43913952f77d5abeee7f95a0c0860",
"sha256:1a4fb8b4ac6772e4d656103ccaf2e43e45bd16b5da324b963d58ef360d09eb73",
"sha256:2e4812720582d0803e84aefa2ac48ce1e1e6e200ca3ce1ae2be6d410c1d637ae",
"sha256:2f01b27d0b453f07cbcff01405cdd007e71f5d6410eb01303a16ba19213e58e4",
"sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863",
"sha256:3c927b6aa682c6d96225e1c797f4a5d0b9f777b327dea912b23471aaf5385376",
"sha256:3d5bb598963ac1f1f5b72dd006adb46ca6203e4fb7269a5b6e1f99e85b07ad38",
"sha256:401508efe02341ae681752a87e8ac9ef76df85ef1a238a7a21786a489d2c983d",
"sha256:4514b19abe6dbd36d6c5d75c54faca24b1ceb3999193c5b1f4b685abeabde3d0",
"sha256:47dfa52bed3097c705451bafd56dac26535545a987b6759fa39da1602349d7ba",
"sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb",
"sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980",
"sha256:55b3187e0bed004533149882ef8c24e954321f3be81f8a9ceffe35099b82a0d0",
"sha256:593f96718ad906e24d6534187fdade28b611f8ed06e27ba972ba48aecec45fc6",
"sha256:5e21032efe673b887464667d09406bab6e16d96b09ad87e80859e3a20b6745b6",
"sha256:60a86b7b2b1c300779167cf595e019e61afcc0e20c4838692983a921db9006ac",
"sha256:619f9bf37cdb4c3dc9d4120d2a1003f5db9446f3618a323219f408f6a9df6725",
"sha256:660b91eca10ee1b44c47843894abe3e6cfd80e50c90dee3123befbf7ca486bd3",
"sha256:67710d220af405f5ce22712fa741d85e8b3ada7a457ea419b038469ba379837c",
"sha256:6caed86cd47ae93915d9031dc04be5283c275e1a2af2ceff33932071f3eeff4d",
"sha256:6d2182e648e79213b3881998b30225b3f4b1f3e681f1c1eaf4cacf19bde1040d",
"sha256:72758c9f785831d9d744af282d54c3e0f9db34f7eae521c33798695464993da2",
"sha256:74c2637d12eaacb503b0b06efdf55199a11b1d7c580bd3dd9dfe84cac97ef2f6",
"sha256:755020d5aeb1b10bffd93d119e7709a2a7475b6ad79c8d5226cea3f76d152ce0",
"sha256:7ccc4727300f223184520a6064c161a90b5d0283accd72d1455bcd85ec44dd0d",
"sha256:81ab21d03e3b0351847a86a0b298b297fde1e152752614138021d6d16a476ea6",
"sha256:8371217dff635cfc0220db2720fc3ce728cd47e72bb7572cca035332823dbdfc",
"sha256:876567136b0359f6581ecd892bdb4ca03a0eead0265db73206c78cff03bcdb0f",
"sha256:879411d04068bd489db57dcf6b82ffad3c5fb2a1fdd30817c566d8b7bedee442",
"sha256:898500957ae5e7f31b7271ace4e6f3625b38c0ac84e8cedde8de3a77a7fdae5e",
"sha256:8c9ca56345b0c5574db47560603de9d05f63cce5dfeb3a456eb60f3fec737ff2",
"sha256:8ec2c146e10b59c376b6bc0369929647fcd95404a503a7aa0990f21c16462248",
"sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576",
"sha256:909bdd4e19ea437eb9b45d6695d722f6f0fd9d8f493e837d70f92062b9f39faf",
"sha256:9d97c713433087ba5cee61a3e8edb54029753d45a4288ad61a176fa4718033ce",
"sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210",
"sha256:aa9087571729c968cd853d54b3f6e9d0ec61e45cd2c31e0eb8a0d4bdbbe6da2f",
"sha256:aef0889417eda2db000d791f9739f5cecb9ccdd45c98f82c6be531bdc67ff0f2",
"sha256:b253d0c53c8ee12c3e53d181fb9ef6ce2cd9c41cbca1c56a535e4fc8ec41e241",
"sha256:b80f6f6478f9d4ca26daee6c61584499493bf97950cfaa1a02b16bb5c2c17e70",
"sha256:be6329b5ba18ec5d32dc26181e0148e423347ed936dda48bf49fb243895d1566",
"sha256:c7560f622e3849cc8f3e999791a915addd08fafe80b47fcf3ffbda5b5151047c",
"sha256:d1a7a716bb04b1c3c4a707e38e2dee46ac544fff931e66d7ae944f3019fc55b8",
"sha256:d63b04e16df8ea21dfcedbf5a60e11cbba9d835d44cb3cbff233cfd037a916d5",
"sha256:d777d239036815e9b3a093fa9208ad314c040c26d7246617e70e23025b60083a",
"sha256:e892d3177380ec080550b56a7ffeab680af25575d291766bdd875147ba246a91",
"sha256:e9c90a44470f2999779057aeaf33461cbd8bb59d8f15e983150d10bb260e16e0",
"sha256:f097dda5d4f9b9b01b3c9fa2069f9c02929365f48f341feddf3d6b32510a2f93",
"sha256:f4ebfe03cbae821ef994b2e58e4df6a087470cc522aca502614e82a143365d45"
],
"markers": "python_version >= '3.6'",
"version": "==0.19.0"
}
},
"develop": {

2
README.md
View File

@ -1,7 +1,7 @@
[![ci](https://github.com/paperless-ngx/paperless-ngx/workflows/ci/badge.svg)](https://github.com/paperless-ngx/paperless-ngx/actions)
[![Crowdin](https://badges.crowdin.net/paperless-ngx/localized.svg)](https://crowdin.com/project/paperless-ngx)
[![Documentation Status](https://img.shields.io/github/deployments/paperless-ngx/paperless-ngx/github-pages?label=docs)](https://docs.paperless-ngx.com)
[![Coverage Status](https://coveralls.io/repos/github/paperless-ngx/paperless-ngx/badge.svg?branch=master)](https://coveralls.io/github/paperless-ngx/paperless-ngx?branch=master)
[![codecov](https://codecov.io/gh/paperless-ngx/paperless-ngx/branch/main/graph/badge.svg?token=VK6OUPJ3TY)](https://codecov.io/gh/paperless-ngx/paperless-ngx)
[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/%23paperlessngx%3Amatrix.org)
[![demo](https://cronitor.io/badges/ve7ItY/production/W5E_B9jkelG9ZbDiNHUPQEVH3MY.svg)](https://demo.paperless-ngx.com)

2
docker/compose/docker-compose.mariadb-tika.yml
View File

@ -59,7 +59,7 @@ services:
- gotenberg
- tika
ports:
- 8000:8000
- "8000:8000"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000"]
interval: 30s

2
docker/compose/docker-compose.mariadb.yml
View File

@ -53,7 +53,7 @@ services:
- db
- broker
ports:
- 8000:8000
- "8000:8000"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000"]
interval: 30s

2
docker/compose/docker-compose.portainer.yml
View File

@ -53,7 +53,7 @@ services:
- db
- broker
ports:
- 8010:8000
- "8010:8000"
healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s

2
docker/compose/docker-compose.postgres-tika.yml
View File

@ -57,7 +57,7 @@ services:
- gotenberg
- tika
ports:
- 8000:8000
- "8000:8000"
healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s

2
docker/compose/docker-compose.postgres.yml
View File

@ -51,7 +51,7 @@ services:
- db
- broker
ports:
- 8000:8000
- "8000:8000"
healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s

2
docker/compose/docker-compose.sqlite-tika.yml
View File

@ -46,7 +46,7 @@ services:
- gotenberg
- tika
ports:
- 8000:8000
- "8000:8000"
healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s

2
docker/compose/docker-compose.sqlite.yml
View File

@ -37,7 +37,7 @@ services:
depends_on:
- broker
ports:
- 8000:8000
- "8000:8000"
healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s

2
docker/docker-prepare.sh
View File

@ -80,7 +80,7 @@ django_checks() {
search_index() {
local -r index_version=2
local -r index_version=3
local -r index_version_file=${DATA_DIR}/.index_version
if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then

7
docker/flower-conditional.sh
View File

@ -3,5 +3,10 @@
echo "Checking if we should start flower..."
if [[ -n "${PAPERLESS_ENABLE_FLOWER}" ]]; then
celery --app paperless flower
# Small delay to allow celery to be up first
echo "Starting flower in 5s"
sleep 5
celery --app paperless flower --conf=/usr/src/paperless/src/paperless/flowerconfig.py
else
echo "Not starting flower"
fi

26
docs/configuration.md
View File

@ -141,7 +141,8 @@ directory.
files created using "collectstatic" manager command are stored.
Unless you're doing something fancy, there is no need to override
this.
this. If this is changed, you may need to run
`collectstatic` again.
Defaults to "../static/", relative to the "src" directory.
@ -757,6 +758,18 @@ should be a valid crontab(5) expression describing when to run.
Defaults to `30 0 * * sun` or Sunday at 30 minutes past midnight.
`PAPERLESS_ENABLE_COMPRESSION=<bool>`
: Enables compression of the responses from the webserver.
: Defaults to 1, enabling compression.
!!! note
If you are using a proxy such as nginx, it is likely more efficient
to enable compression in your proxy configuration rather than
the webserver
## Polling {#polling}
`PAPERLESS_CONSUMER_POLLING=<num>`
@ -986,13 +999,20 @@ within your documents.
`PAPERLESS_CONSUMER_IGNORE_PATTERNS=<json>`
: By default, paperless ignores certain files and folders in the
consumption directory, such as system files created by the Mac OS.
consumption directory, such as system files created by the Mac OS
or hidden folders some tools use to store data.
This can be adjusted by configuring a custom json array with
patterns to exclude.
For example, `.DS_STORE/*` will ignore any files found in a folder
named `.DS_STORE`, including `.DS_STORE/bar.pdf` and `foo/.DS_STORE/bar.pdf`
A pattern like `._*` will ignore anything starting with `._`, including:
`._foo.pdf` and `._bar/foo.pdf`
Defaults to
`[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]`.
`[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]`.
## Binaries

4
install-paperless-ngx.sh
View File

@ -321,7 +321,7 @@ fi
wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/docker-compose.$DOCKER_COMPOSE_VERSION.yml" -O docker-compose.yml
wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/.env" -O .env
SECRET_KEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 64 | head -n 1)
SECRET_KEY=$(tr --delete --complement 'a-zA-Z0-9' < /dev/urandom 2>/dev/null | head --bytes 64)
DEFAULT_LANGUAGES=("deu eng fra ita spa")
@ -346,7 +346,7 @@ read -r -a OCR_LANGUAGES_ARRAY <<< "${_split_langs}"
fi
} > docker-compose.env
sed -i "s/- 8000:8000/- $PORT:8000/g" docker-compose.yml
sed -i "s/- \"8000:8000\"/- \"$PORT:8000\"/g" docker-compose.yml
sed -i "s#- \./consume:/usr/src/paperless/consume#- $CONSUME_FOLDER:/usr/src/paperless/consume#g" docker-compose.yml

3
src-ui/angular.json
View File

@ -192,7 +192,8 @@
"cli": {
"schematicCollections": [
"@angular-eslint/schematics"
]
],
"analytics": false
},
"schematics": {
"@angular-eslint/schematics:application": {

151
src-ui/messages.xlf
View File

@ -445,6 +445,10 @@
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">1</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">192</context>
</context-group>
</trans-unit>
<trans-unit id="3797778920049399855" datatype="html">
<source>Logout</source>
@ -711,7 +715,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">492</context>
<context context-type="linenumber">498</context>
</context-group>
</trans-unit>
<trans-unit id="2526035785704676448" datatype="html">
@ -879,15 +883,15 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">196</context>
<context context-type="linenumber">204</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">248</context>
<context context-type="linenumber">256</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">283</context>
<context context-type="linenumber">291</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context>
@ -1026,7 +1030,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">317</context>
<context context-type="linenumber">325</context>
</context-group>
</trans-unit>
<trans-unit id="6457471243969293847" datatype="html">
@ -1163,7 +1167,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">284</context>
<context context-type="linenumber">292</context>
</context-group>
</trans-unit>
<trans-unit id="7046259383943324039" datatype="html">
@ -1401,15 +1405,15 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">214</context>
<context context-type="linenumber">222</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">261</context>
<context context-type="linenumber">269</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">296</context>
<context context-type="linenumber">304</context>
</context-group>
</trans-unit>
<trans-unit id="2784260611081866636" datatype="html">
@ -1703,11 +1707,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">222</context>
<context context-type="linenumber">230</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">308</context>
<context context-type="linenumber">316</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context>
@ -2031,7 +2035,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">32</context>
<context context-type="linenumber">34</context>
</context-group>
</trans-unit>
<trans-unit id="4452427314943113135" datatype="html">
@ -2269,7 +2273,7 @@
<source>Confirm delete</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">449</context>
<context context-type="linenumber">453</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/management-list/management-list.component.ts</context>
@ -2280,35 +2284,35 @@
<source>Do you really want to delete document &quot;<x id="PH" equiv-text="this.document.title"/>&quot;?</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">450</context>
<context context-type="linenumber">454</context>
</context-group>
</trans-unit>
<trans-unit id="6691075929777935948" datatype="html">
<source>The files for this document will be deleted permanently. This operation cannot be undone.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">451</context>
<context context-type="linenumber">455</context>
</context-group>
</trans-unit>
<trans-unit id="719892092227206532" datatype="html">
<source>Delete document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">453</context>
<context context-type="linenumber">457</context>
</context-group>
</trans-unit>
<trans-unit id="1844801255494293730" datatype="html">
<source>Error deleting document: <x id="PH" equiv-text="JSON.stringify(error)"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">469</context>
<context context-type="linenumber">473</context>
</context-group>
</trans-unit>
<trans-unit id="7362691899087997122" datatype="html">
<source>Redo OCR confirm</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">489</context>
<context context-type="linenumber">493</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@ -2319,14 +2323,14 @@
<source>This operation will permanently redo OCR for this document.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">490</context>
<context context-type="linenumber">494</context>
</context-group>
</trans-unit>
<trans-unit id="5641451190833696892" datatype="html">
<source>This operation cannot be undone.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">491</context>
<context context-type="linenumber">495</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@ -2338,18 +2342,18 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">576</context>
<context context-type="linenumber">582</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">635</context>
<context context-type="linenumber">641</context>
</context-group>
</trans-unit>
<trans-unit id="1181910457994920507" datatype="html">
<source>Proceed</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">493</context>
<context context-type="linenumber">497</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@ -2357,18 +2361,18 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">578</context>
<context context-type="linenumber">584</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">637</context>
<context context-type="linenumber">643</context>
</context-group>
</trans-unit>
<trans-unit id="5729001209753056399" datatype="html">
<source>Redo OCR operation will begin in the background. Close and re-open or reload this document after the operation has completed to see new content.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">501</context>
<context context-type="linenumber">505</context>
</context-group>
</trans-unit>
<trans-unit id="8008978164775353960" datatype="html">
@ -2377,7 +2381,7 @@
)"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">512,514</context>
<context context-type="linenumber">516,518</context>
</context-group>
</trans-unit>
<trans-unit id="6857598786757174736" datatype="html">
@ -2462,15 +2466,15 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">213</context>
<context context-type="linenumber">221</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">250</context>
<context context-type="linenumber">258</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">285</context>
<context context-type="linenumber">293</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context>
@ -2748,11 +2752,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">260</context>
<context context-type="linenumber">268</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">295</context>
<context context-type="linenumber">303</context>
</context-group>
</trans-unit>
<trans-unit id="2509141182388535183" datatype="html">
@ -2886,6 +2890,10 @@
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
<context context-type="linenumber">64</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">199</context>
</context-group>
</trans-unit>
<trans-unit id="1233494216161906927" datatype="html">
<source>Save &quot;<x id="INTERPOLATION" equiv-text="{{list.activeSavedViewTitle}}"/>&quot;</source>
@ -3114,7 +3122,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">208</context>
<context context-type="linenumber">216</context>
</context-group>
</trans-unit>
<trans-unit id="4104807402967139762" datatype="html">
@ -3125,7 +3133,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">204</context>
<context context-type="linenumber">212</context>
</context-group>
</trans-unit>
<trans-unit id="6965614903949668392" datatype="html">
@ -3647,123 +3655,130 @@
<context context-type="linenumber">181</context>
</context-group>
</trans-unit>
<trans-unit id="1595668988802980095" datatype="html">
<source>Show warning when closing saved views with unsaved changes</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">195</context>
</context-group>
</trans-unit>
<trans-unit id="6925788033494878061" datatype="html">
<source>Appears on</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">201</context>
<context context-type="linenumber">209</context>
</context-group>
</trans-unit>
<trans-unit id="7877440816920439876" datatype="html">
<source>No saved views defined.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">218</context>
<context context-type="linenumber">226</context>
</context-group>
</trans-unit>
<trans-unit id="1292737233370901804" datatype="html">
<source>Mail</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">232,231</context>
<context context-type="linenumber">240,239</context>
</context-group>
</trans-unit>
<trans-unit id="8913167930428886792" datatype="html">
<source>Mail accounts</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">236</context>
<context context-type="linenumber">244</context>
</context-group>
</trans-unit>
<trans-unit id="1259421956660976189" datatype="html">
<source>Add Account</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">241</context>
<context context-type="linenumber">249</context>
</context-group>
</trans-unit>
<trans-unit id="2188854519574316630" datatype="html">
<source>Server</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">249</context>
<context context-type="linenumber">257</context>
</context-group>
</trans-unit>
<trans-unit id="6235247415162820954" datatype="html">
<source>No mail accounts defined.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">267</context>
<context context-type="linenumber">275</context>
</context-group>
</trans-unit>
<trans-unit id="5364020217520256833" datatype="html">
<source>Mail rules</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">271</context>
<context context-type="linenumber">279</context>
</context-group>
</trans-unit>
<trans-unit id="1372022816709469401" datatype="html">
<source>Add Rule</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">276</context>
<context context-type="linenumber">284</context>
</context-group>
</trans-unit>
<trans-unit id="6751234988479444294" datatype="html">
<source>No mail rules defined.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">302</context>
<context context-type="linenumber">310</context>
</context-group>
</trans-unit>
<trans-unit id="5610279464668232148" datatype="html">
<source>Saved view &quot;<x id="PH" equiv-text="savedView.name"/>&quot; deleted.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">383</context>
<context context-type="linenumber">385</context>
</context-group>
</trans-unit>
<trans-unit id="3891152409365583719" datatype="html">
<source>Settings saved</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">476</context>
<context context-type="linenumber">482</context>
</context-group>
</trans-unit>
<trans-unit id="7217000812750597833" datatype="html">
<source>Settings were saved successfully.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">477</context>
<context context-type="linenumber">483</context>
</context-group>
</trans-unit>
<trans-unit id="525012668859298131" datatype="html">
<source>Settings were saved successfully. Reload is required to apply some changes.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">481</context>
<context context-type="linenumber">487</context>
</context-group>
</trans-unit>
<trans-unit id="8491974984518503778" datatype="html">
<source>Reload now</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">482</context>
<context context-type="linenumber">488</context>
</context-group>
</trans-unit>
<trans-unit id="6839066544204061364" datatype="html">
<source>Use system language</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">500</context>
<context context-type="linenumber">506</context>
</context-group>
</trans-unit>
<trans-unit id="7729897675462249787" datatype="html">
<source>Use date format of display language</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">507</context>
<context context-type="linenumber">513</context>
</context-group>
</trans-unit>
<trans-unit id="8488620293789898901" datatype="html">
@ -3772,91 +3787,91 @@
)"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">527,529</context>
<context context-type="linenumber">533,535</context>
</context-group>
</trans-unit>
<trans-unit id="6327501535846658797" datatype="html">
<source>Saved account &quot;<x id="PH" equiv-text="newMailAccount.name"/>&quot;.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">554</context>
<context context-type="linenumber">560</context>
</context-group>
</trans-unit>
<trans-unit id="6428427497555765743" datatype="html">
<source>Error saving account: <x id="PH" equiv-text="e.toString()"/>.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">564</context>
<context context-type="linenumber">570</context>
</context-group>
</trans-unit>
<trans-unit id="5641934153807844674" datatype="html">
<source>Confirm delete mail account</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">574</context>
<context context-type="linenumber">580</context>
</context-group>
</trans-unit>
<trans-unit id="7176985344323395435" datatype="html">
<source>This operation will permanently delete this mail account.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">575</context>
<context context-type="linenumber">581</context>
</context-group>
</trans-unit>
<trans-unit id="4233826387148482123" datatype="html">
<source>Deleted mail account</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">584</context>
<context context-type="linenumber">590</context>
</context-group>
</trans-unit>
<trans-unit id="7443801450153832973" datatype="html">
<source>Error deleting mail account: <x id="PH" equiv-text="e.toString()"/>.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">593</context>
<context context-type="linenumber">599</context>
</context-group>
</trans-unit>
<trans-unit id="123368655395433699" datatype="html">
<source>Saved rule &quot;<x id="PH" equiv-text="newMailRule.name"/>&quot;.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">612</context>
<context context-type="linenumber">618</context>
</context-group>
</trans-unit>
<trans-unit id="4741216051394823471" datatype="html">
<source>Error saving rule: <x id="PH" equiv-text="e.toString()"/>.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">623</context>
<context context-type="linenumber">629</context>
</context-group>
</trans-unit>
<trans-unit id="3896080636020672118" datatype="html">
<source>Confirm delete mail rule</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">633</context>
<context context-type="linenumber">639</context>
</context-group>
</trans-unit>
<trans-unit id="2250372580580310337" datatype="html">
<source>This operation will permanently delete this mail rule.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">634</context>
<context context-type="linenumber">640</context>
</context-group>
</trans-unit>
<trans-unit id="9077981247971516916" datatype="html">
<source>Deleted mail rule</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">643</context>
<context context-type="linenumber">649</context>
</context-group>
</trans-unit>
<trans-unit id="4740074357089345173" datatype="html">
<source>Error deleting mail rule: <x id="PH" equiv-text="e.toString()"/>.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">652</context>
<context context-type="linenumber">658</context>
</context-group>
</trans-unit>
<trans-unit id="5101757640976222639" datatype="html">
@ -4113,7 +4128,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">24</context>
<context context-type="linenumber">26</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/services/open-documents.service.ts</context>
@ -4153,21 +4168,21 @@
<source>You have unsaved changes to the saved view</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">26</context>
<context context-type="linenumber">28</context>
</context-group>
</trans-unit>
<trans-unit id="7282050913165342352" datatype="html">
<source>Are you sure you want to close this saved view?</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">30</context>
<context context-type="linenumber">32</context>
</context-group>
</trans-unit>
<trans-unit id="856284624775342512" datatype="html">
<source>Save and close</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">34</context>
<context context-type="linenumber">36</context>
</context-group>
</trans-unit>
<trans-unit id="7536524521722799066" datatype="html">

14779
src-ui/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

34
src-ui/package.json
View File

@ -13,14 +13,14 @@
},
"private": true,
"dependencies": {
"@angular/common": "~15.1.0",
"@angular/compiler": "~15.1.0",
"@angular/core": "~15.1.0",
"@angular/forms": "~15.1.0",
"@angular/localize": "~15.1.0",
"@angular/platform-browser": "~15.1.0",
"@angular/platform-browser-dynamic": "~15.1.0",
"@angular/router": "~15.1.0",
"@angular/common": "~15.1.2",
"@angular/compiler": "~15.1.2",
"@angular/core": "~15.1.2",
"@angular/forms": "~15.1.2",
"@angular/localize": "~15.1.2",
"@angular/platform-browser": "~15.1.2",
"@angular/platform-browser-dynamic": "~15.1.2",
"@angular/router": "~15.1.2",
"@ng-bootstrap/ng-bootstrap": "^14.0.1",
"@ng-select/ng-select": "^10.0.1",
"@ngneat/dirty-check-forms": "^3.0.3",
@ -39,18 +39,18 @@
},
"devDependencies": {
"@angular-builders/jest": "15.0.0",
"@angular-devkit/build-angular": "~15.1.0",
"@angular-eslint/builder": "15.1.0",
"@angular-eslint/eslint-plugin": "15.1.0",
"@angular-eslint/eslint-plugin-template": "15.1.0",
"@angular-eslint/schematics": "15.1.0",
"@angular-eslint/template-parser": "15.1.0",
"@angular/cli": "~15.1.0",
"@angular/compiler-cli": "~15.1.0",
"@angular-devkit/build-angular": "~15.1.4",
"@angular-eslint/builder": "15.2.0",
"@angular-eslint/eslint-plugin": "15.2.0",
"@angular-eslint/eslint-plugin-template": "15.2.0",
"@angular-eslint/schematics": "15.2.0",
"@angular-eslint/template-parser": "15.2.0",
"@angular/cli": "~15.1.4",
"@angular/compiler-cli": "~15.1.2",
"@types/jest": "28.1.6",
"@types/node": "^18.7.23",
"@typescript-eslint/eslint-plugin": "^5.43.0",
"@typescript-eslint/parser": "^5.43.0",
"@typescript-eslint/parser": "^5.50.0",
"concurrently": "7.4.0",
"eslint": "^8.31.0",
"jest": "28.1.3",

2
src-ui/src/app/components/common/input/tags/tags.component.html
View File

@ -17,7 +17,7 @@
(blur)="onBlur()">
<ng-template ng-label-tmp let-item="item">
<span class="tag-wrap tag-wrap-delete" (click)="removeTag(item.id)">
<span class="tag-wrap tag-wrap-delete" (mousedown)="removeTag($event, item.id)">
<svg width="1.2em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
<use xlink:href="assets/bootstrap-icons.svg#x"/>
</svg>

7
src-ui/src/app/components/common/input/tags/tags.component.ts
View File

@ -65,7 +65,7 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
private _lastSearchTerm: string
getTag(id) {
getTag(id: number) {
if (this.tags) {
return this.tags.find((tag) => tag.id == id)
} else {
@ -73,7 +73,10 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
}
}
removeTag(id) {
removeTag(event: PointerEvent, id: number) {
// prevent opening dropdown
event.stopImmediatePropagation()
let index = this.value.indexOf(id)
if (index > -1) {
let oldValue = this.value

2
src-ui/src/app/components/document-detail/document-detail.component.html
View File

@ -63,7 +63,7 @@
<div class="row">
<div class="col mb-4">
<div class="col-md-6 col-xl-4 mb-4">
<form [formGroup]='documentForm' (ngSubmit)="save()">

9
src-ui/src/app/components/document-detail/document-detail.component.scss
View File

@ -22,6 +22,15 @@
--page-margin: 1px 0 20px;
}
::ng-deep .ng-select-taggable {
max-width: calc(100% - 46px); // fudge factor for ng-select button width
}
.btn-group .dropdown-toggle-split {
border-top-right-radius: inherit;
border-bottom-right-radius: inherit;
}
.password-prompt {
position: absolute;
top: 30%;

4
src-ui/src/app/components/manage/management-list/management-list.component.html
View File

@ -10,7 +10,7 @@
</div>
</div>
<ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
<ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" [maxSize]="5" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
</div>
<table class="table table-striped align-middle border shadow-sm">
@ -72,5 +72,5 @@
<div class="d-flex">
<div i18n *ngIf="collectionSize > 0">{collectionSize, plural, =1 {One {{typeName}}} other {{{collectionSize || 0}} total {{typeNamePlural}}}}</div>
<ngb-pagination *ngIf="collectionSize > 20" class="ms-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
<ngb-pagination *ngIf="collectionSize > 20" class="ms-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" [maxSize]="5" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
</div>

8
src-ui/src/app/components/manage/settings/settings.component.html
View File

@ -189,6 +189,14 @@
<a ngbNavLink i18n>Saved views</a>
<ng-template ngbNavContent>
<h4 i18n>Settings</h4>
<div class="row mb-3">
<div class="offset-md-3 col">
<app-input-check i18n-title title="Show warning when closing saved views with unsaved changes" formControlName="savedViewsWarnOnUnsavedChange"></app-input-check>
</div>
</div>
<h4 i18n>Views</h4>
<div formGroupName="savedViews">
<div *ngFor="let view of savedViews" [formGroupName]="view.id" class="row">

8
src-ui/src/app/components/manage/settings/settings.component.ts
View File

@ -84,6 +84,7 @@ export class SettingsComponent
notificationsConsumerFailed: new FormControl(null),
notificationsConsumerSuppressOnDashboard: new FormControl(null),
savedViewsWarnOnUnsavedChange: new FormControl(null),
savedViews: this.savedViewGroup,
mailAccounts: this.mailAccountGroup,
@ -194,6 +195,9 @@ export class SettingsComponent
notificationsConsumerSuppressOnDashboard: this.settings.get(
SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD
),
savedViewsWarnOnUnsavedChange: this.settings.get(
SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE
),
savedViews: {},
mailAccounts: {},
mailRules: {},
@ -462,6 +466,10 @@ export class SettingsComponent
SETTINGS_KEYS.UPDATE_CHECKING_ENABLED,
this.settingsForm.value.updateCheckingEnabled
)
this.settings.set(
SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE,
this.settingsForm.value.savedViewsWarnOnUnsavedChange
)
this.settings.setLanguage(this.settingsForm.value.displayLanguage)
this.settings
.storeSettings()

7
src-ui/src/app/data/paperless-uisettings.ts
View File

@ -41,6 +41,8 @@ export const SETTINGS_KEYS = {
UPDATE_CHECKING_ENABLED: 'general-settings:update-checking:enabled',
UPDATE_CHECKING_BACKEND_SETTING:
'general-settings:update-checking:backend-setting',
SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE:
'general-settings:saved-views:warn-on-unsaved-change',
}
export const SETTINGS: PaperlessUiSetting[] = [
@ -139,4 +141,9 @@ export const SETTINGS: PaperlessUiSetting[] = [
type: 'string',
default: '',
},
{
key: SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE,
type: 'boolean',
default: true,
},
]

12
src-ui/src/app/guards/dirty-saved-view.guard.ts
View File

@ -4,17 +4,25 @@ import { first, Observable, Subject } from 'rxjs'
import { DocumentListComponent } from '../components/document-list/document-list.component'
import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
import { ConfirmDialogComponent } from '../components/common/confirm-dialog/confirm-dialog.component'
import { SettingsService } from '../services/settings.service'
import { SETTINGS_KEYS } from '../data/paperless-uisettings'
@Injectable()
export class DirtySavedViewGuard
implements CanDeactivate<DocumentListComponent>
{
constructor(private modalService: NgbModal) {}
constructor(
private modalService: NgbModal,
private settings: SettingsService
) {}
canDeactivate(
component: DocumentListComponent
): boolean | Observable<boolean> {
return component.savedViewIsModified ? this.warn(component) : true
return component.savedViewIsModified &&
this.settings.get(SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE)
? this.warn(component)
: true
}
warn(component: DocumentListComponent) {

2
src-ui/src/environments/environment.prod.ts
View File

@ -5,7 +5,7 @@ export const environment = {
apiBaseUrl: document.baseURI + 'api/',
apiVersion: '2',
appTitle: 'Paperless-ngx',
version: '1.12.2',
version: '1.12.2-dev',
webSocketHost: window.location.host,
webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
webSocketBaseUrl: base_url.pathname + 'ws/',

545
src-ui/src/locale/messages.ar_AR.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.be_BY.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.cs_CZ.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.da_DK.xlf
View File

File diff suppressed because it is too large Load Diff

543
src-ui/src/locale/messages.de_DE.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.es_ES.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.fi_FI.xlf
View File

File diff suppressed because it is too large Load Diff

867
src-ui/src/locale/messages.fr_FR.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.he_IL.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.hr_HR.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.it_IT.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.lb_LU.xlf
View File

File diff suppressed because it is too large Load Diff

837
src-ui/src/locale/messages.nl_NL.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.no_NO.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.pl_PL.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.pt_BR.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.pt_PT.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.ro_RO.xlf
View File

File diff suppressed because it is too large Load Diff

819
src-ui/src/locale/messages.ru_RU.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.sl_SI.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.sr_CS.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.sv_SE.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.tr_TR.xlf
View File

File diff suppressed because it is too large Load Diff

545
src-ui/src/locale/messages.zh_CN.xlf
View File

File diff suppressed because it is too large Load Diff

64
src/documents/barcodes.py
View File

@ -5,6 +5,7 @@ import tempfile
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import Dict
from typing import List
from typing import Optional
@ -201,16 +202,25 @@ def scan_file_for_barcodes(
return DocumentBarcodeInfo(pdf_filepath, barcodes)
def get_separating_barcodes(barcodes: List[Barcode]) -> List[int]:
def get_separating_barcodes(barcodes: List[Barcode]) -> Dict[int, bool]:
"""
Search the parsed barcodes for separators
and returns a list of page numbers, which
separate the file into new files.
and returns a dict of page numbers, which
separate the file into new files, together
with the information whether to keep the page.
"""
# filter all barcodes for the separator string
# get the page numbers of the separating barcodes
separator_pages = {bc.page: False for bc in barcodes if bc.is_separator}
if not settings.CONSUMER_ENABLE_ASN_BARCODE:
return separator_pages
return list({bc.page for bc in barcodes if bc.is_separator})
# add the page numbers of the ASN barcodes
# (except for first page, that might lead to infinite loops).
return {
**separator_pages,
**{bc.page: True for bc in barcodes if bc.is_asn and bc.page != 0},
}
def get_asn_from_barcodes(barcodes: List[Barcode]) -> Optional[int]:
@ -242,10 +252,11 @@ def get_asn_from_barcodes(barcodes: List[Barcode]) -> Optional[int]:
return asn
def separate_pages(filepath: str, pages_to_split_on: List[int]) -> List[str]:
def separate_pages(filepath: str, pages_to_split_on: Dict[int, bool]) -> List[str]:
"""
Separate the provided pdf file on the pages_to_split_on.
The pages which are defined by page_numbers will be removed.
The pages which are defined by the keys in page_numbers
will be removed if the corresponding value is false.
Returns a list of (temporary) filepaths to consume.
These will need to be deleted later.
"""
@ -261,26 +272,28 @@ def separate_pages(filepath: str, pages_to_split_on: List[int]) -> List[str]:
fname = os.path.splitext(os.path.basename(filepath))[0]
pdf = Pdf.open(filepath)
# Start with an empty document
current_document: List[Page] = []
# A list of documents, ie a list of lists of pages
documents: List[List[Page]] = []
# A single document, ie a list of pages
document: List[Page] = []
documents: List[List[Page]] = [current_document]
for idx, page in enumerate(pdf.pages):
# Keep building the new PDF as long as it is not a
# separator index
if idx not in pages_to_split_on:
document.append(page)
# Make sure to append the very last document to the documents
if idx == (len(pdf.pages) - 1):
documents.append(document)
document = []
else:
# This is a split index, save the current PDF pages, and restart
# a new destination page listing
logger.debug(f"Starting new document at idx {idx}")
documents.append(document)
document = []
current_document.append(page)
continue
# This is a split index
# Start a new destination page listing
logger.debug(f"Starting new document at idx {idx}")
current_document = []
documents.append(current_document)
keep_page = pages_to_split_on[idx]
if keep_page:
# Keep the page
# (new document is started by asn barcode)
current_document.append(page)
documents = [x for x in documents if len(x)]
@ -312,11 +325,10 @@ def save_to_dir(
Optionally rename the file.
"""
if os.path.isfile(filepath) and os.path.isdir(target_dir):
dst = shutil.copy(filepath, target_dir)
logging.debug(f"saved {str(filepath)} to {str(dst)}")
if newname:
dst_new = os.path.join(target_dir, newname)
logger.debug(f"moving {str(dst)} to {str(dst_new)}")
os.rename(dst, dst_new)
dest = target_dir
if newname is not None:
dest = os.path.join(dest, newname)
shutil.copy(filepath, dest)
logging.debug(f"saved {str(filepath)} to {str(dest)}")
else:
logger.warning(f"{str(filepath)} or {str(target_dir)} don't exist.")

11
src/documents/consumer.py
View File

@ -146,11 +146,16 @@ class Consumer(LoggingMixin):
return
# Validate the range is above zero and less than uint32_t max
# otherwise, Whoosh can't handle it in the index
if self.override_asn < 0 or self.override_asn > 0xFF_FF_FF_FF:
if (
self.override_asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
or self.override_asn > Document.ARCHIVE_SERIAL_NUMBER_MAX
):
self._fail(
MESSAGE_ASN_RANGE,
f"Not consuming {self.filename}: "
f"Given ASN {self.override_asn} is out of range [0, 4,294,967,295]",
f"Given ASN {self.override_asn} is out of range "
f"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, "
f"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}]",
)
if Document.objects.filter(archive_serial_number=self.override_asn).exists():
self._fail(
@ -337,6 +342,7 @@ class Consumer(LoggingMixin):
mime_type,
)
if not parser_class:
tempdir.cleanup()
self._fail(MESSAGE_UNSUPPORTED_TYPE, f"Unsupported mime type {mime_type}")
# Notify all listeners that we're going to do some work.
@ -395,6 +401,7 @@ class Consumer(LoggingMixin):
except ParseError as e:
document_parser.cleanup()
tempdir.cleanup()
self._fail(
str(e),
f"Error while consuming document {self.filename}: {e}",

19
src/documents/index.py
View File

@ -5,6 +5,7 @@ from contextlib import contextmanager
from dateutil.parser import isoparse
from django.conf import settings
from django.utils import timezone
from documents.models import Comment
from documents.models import Document
from whoosh import classify
@ -89,10 +90,22 @@ def open_index_searcher():
searcher.close()
def update_document(writer, doc):
def update_document(writer: AsyncWriter, doc: Document):
tags = ",".join([t.name for t in doc.tags.all()])
tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
comments = ",".join([str(c.comment) for c in Comment.objects.filter(document=doc)])
asn = doc.archive_serial_number
if asn is not None and (
asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
or asn > Document.ARCHIVE_SERIAL_NUMBER_MAX
):
logger.error(
f"Not indexing Archive Serial Number {asn} of document {doc.pk}. "
f"ASN is out of range "
f"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, "
f"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}.",
)
asn = 0
writer.update_document(
id=doc.pk,
title=doc.title,
@ -108,7 +121,7 @@ def update_document(writer, doc):
has_type=doc.document_type is not None,
created=doc.created,
added=doc.added,
asn=doc.archive_serial_number,
asn=asn,
modified=doc.modified,
path=doc.storage_path.name if doc.storage_path else None,
path_id=doc.storage_path.id if doc.storage_path else None,
@ -262,7 +275,7 @@ class DelayedFullTextQuery(DelayedQuery):
["content", "title", "correspondent", "tag", "type", "comments"],
self.searcher.ixreader.schema,
)
qp.add_plugin(DateParserPlugin())
qp.add_plugin(DateParserPlugin(basedate=timezone.now()))
q = qp.parse(q_str)
corrected = self.searcher.correct_query(q, q_str)

61
src/documents/management/commands/document_consumer.py
View File

@ -1,5 +1,6 @@
import logging
import os
from fnmatch import filter
from pathlib import Path
from pathlib import PurePath
from threading import Event
@ -7,6 +8,7 @@ from threading import Thread
from time import monotonic
from time import sleep
from typing import Final
from typing import Set
from django.conf import settings
from django.core.management.base import BaseCommand
@ -25,15 +27,15 @@ except ImportError: # pragma: nocover
logger = logging.getLogger("paperless.management.consumer")
def _tags_from_path(filepath):
"""Walk up the directory tree from filepath to CONSUMPTION_DIR
and get or create Tag IDs for every directory.
def _tags_from_path(filepath) -> Set[Tag]:
"""
Walk up the directory tree from filepath to CONSUMPTION_DIR
and get or create Tag IDs for every directory.
Returns set of Tag models
"""
normalized_consumption_dir = os.path.abspath(
os.path.normpath(settings.CONSUMPTION_DIR),
)
tag_ids = set()
path_parts = Path(filepath).relative_to(normalized_consumption_dir).parent.parts
path_parts = Path(filepath).relative_to(settings.CONSUMPTION_DIR).parent.parts
for part in path_parts:
tag_ids.add(
Tag.objects.get_or_create(name__iexact=part, defaults={"name": part})[0].pk,
@ -43,14 +45,41 @@ def _tags_from_path(filepath):
def _is_ignored(filepath: str) -> bool:
normalized_consumption_dir = os.path.abspath(
os.path.normpath(settings.CONSUMPTION_DIR),
"""
Checks if the given file should be ignored, based on configured
patterns.
Returns True if the file is ignored, False otherwise
"""
filepath = os.path.abspath(
os.path.normpath(filepath),
)
filepath_relative = PurePath(filepath).relative_to(normalized_consumption_dir)
return any(filepath_relative.match(p) for p in settings.CONSUMER_IGNORE_PATTERNS)
# Trim out the consume directory, leaving only filename and it's
# path relative to the consume directory
filepath_relative = PurePath(filepath).relative_to(settings.CONSUMPTION_DIR)
# March through the components of the path, including directories and the filename
# looking for anything matching
# foo/bar/baz/file.pdf -> (foo, bar, baz, file.pdf)
parts = []
for part in filepath_relative.parts:
# If the part is not the name (ie, it's a dir)
# Need to append the trailing slash or fnmatch doesn't match
# fnmatch("dir", "dir/*") == False
# fnmatch("dir/", "dir/*") == True
if part != filepath_relative.name:
part = part + "/"
parts.append(part)
for pattern in settings.CONSUMER_IGNORE_PATTERNS:
if len(filter(parts, pattern)):
return True
return False
def _consume(filepath):
def _consume(filepath: str) -> None:
if os.path.isdir(filepath) or _is_ignored(filepath):
return
@ -103,7 +132,13 @@ def _consume(filepath):
logger.exception("Error while consuming document")
def _consume_wait_unmodified(file):
def _consume_wait_unmodified(file: str) -> None:
"""
Waits for the given file to appear unmodified based on file size
and modification time. Will wait a configured number of seconds
and retry a configured number of times before either consuming or
giving up
"""
if _is_ignored(file):
return

2
src/documents/management/commands/document_exporter.py
View File

@ -311,8 +311,8 @@ class Command(BaseCommand):
archive_target = None
# 3.4. write files to target folder
t = int(time.mktime(document.created.timetuple()))
if document.storage_type == Document.STORAGE_TYPE_GPG:
t = int(time.mktime(document.created.timetuple()))
original_target.parent.mkdir(parents=True, exist_ok=True)
with document.source_file as out_file:

23
src/documents/migrations/1030_alter_paperlesstask_task_file_name.py Normal file
View File

@ -0,0 +1,23 @@
# Generated by Django 4.1.5 on 2023-02-03 21:53
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("documents", "1029_alter_document_archive_serial_number"),
]
operations = [
migrations.AlterField(
model_name="paperlesstask",
name="task_file_name",
field=models.CharField(
help_text="Name of the file which the Task was run for",
max_length=255,
null=True,
verbose_name="Task Filename",
),
),
]

10
src/documents/models.py
View File

@ -3,6 +3,7 @@ import logging
import os
import re
from collections import OrderedDict
from typing import Final
from typing import Optional
import dateutil.parser
@ -229,6 +230,9 @@ class Document(models.Model):
help_text=_("The original name of the file when it was uploaded"),
)
ARCHIVE_SERIAL_NUMBER_MIN: Final[int] = 0
ARCHIVE_SERIAL_NUMBER_MAX: Final[int] = 0xFF_FF_FF_FF
archive_serial_number = models.PositiveIntegerField(
_("archive serial number"),
blank=True,
@ -236,8 +240,8 @@ class Document(models.Model):
unique=True,
db_index=True,
validators=[
MaxValueValidator(0xFF_FF_FF_FF),
MinValueValidator(0),
MaxValueValidator(ARCHIVE_SERIAL_NUMBER_MAX),
MinValueValidator(ARCHIVE_SERIAL_NUMBER_MIN),
],
help_text=_(
"The position of this document in your physical document " "archive.",
@ -555,7 +559,7 @@ class PaperlessTask(models.Model):
task_file_name = models.CharField(
null=True,
max_length=255,
verbose_name=_("Task Name"),
verbose_name=_("Task Filename"),
help_text=_("Name of the file which the Task was run for"),
)

6
src/documents/serialisers.py
View File

@ -599,11 +599,17 @@ class StoragePathSerializer(MatchingModelSerializer):
document_type="document_type",
created="created",
created_year="created_year",
created_year_short="created_year_short",
created_month="created_month",
created_month_name="created_month_name",
created_month_name_short="created_month_name_short",
created_day="created_day",
added="added",
added_year="added_year",
added_year_short="added_year_short",
added_month="added_month",
added_month_name="added_month_name",
added_month_name_short="added_month_name_short",
added_day="added_day",
asn="asn",
tags="tags",

31
src/documents/tasks.py
View File

@ -128,6 +128,18 @@ def consume_file(
)
if document_list:
# If the file is an upload, it's in the scratch directory
# Move it to consume directory to be picked up
# Otherwise, use the current parent to keep possible tags
# from subdirectories
try:
# is_relative_to would be nicer, but new in 3.9
_ = path.relative_to(settings.SCRATCH_DIR)
save_to_dir = settings.CONSUMPTION_DIR
except ValueError:
save_to_dir = path.parent
for n, document in enumerate(document_list):
# save to consumption dir
# rename it to the original filename with number prefix
@ -136,23 +148,18 @@ def consume_file(
else:
newname = None
# If the file is an upload, it's in the scratch directory
# Move it to consume directory to be picked up
# Otherwise, use the current parent to keep possible tags
# from subdirectories
try:
# is_relative_to would be nicer, but new in 3.9
_ = path.relative_to(settings.SCRATCH_DIR)
save_to_dir = settings.CONSUMPTION_DIR
except ValueError:
save_to_dir = path.parent
barcodes.save_to_dir(
document,
newname=newname,
target_dir=save_to_dir,
)
# Split file has been copied safely, remove it
os.remove(document)
# And clean up the directory as well, now it's empty
shutil.rmtree(os.path.dirname(document_list[0]))
# Delete the PDF file which was split
os.remove(doc_barcode_info.pdf_path)
@ -164,7 +171,7 @@ def consume_file(
# notify the sender, otherwise the progress bar
# in the UI stays stuck
payload = {
"filename": override_filename,
"filename": override_filename or path.name,
"task_id": task_id,
"current_progress": 100,
"max_progress": 100,

BIN
src/documents/tests/samples/barcodes/split-by-asn-1.pdf Normal file
View File

Binary file not shown.

BIN
src/documents/tests/samples/barcodes/split-by-asn-2.pdf Normal file
View File

Binary file not shown.

306
src/documents/tests/test_api.py
View File

@ -7,6 +7,7 @@ import tempfile
import urllib.request
import uuid
import zipfile
from datetime import timedelta
from pathlib import Path
from unittest import mock
from unittest.mock import MagicMock
@ -23,6 +24,7 @@ from django.conf import settings
from django.contrib.auth.models import User
from django.test import override_settings
from django.utils import timezone
from dateutil.relativedelta import relativedelta
from documents import bulk_edit
from documents import index
from documents.models import Correspondent
@ -119,28 +121,28 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
response = self.client.get("/api/documents/", format="json")
self.assertEqual(response.status_code, 200)
results_full = response.data["results"]
self.assertTrue("content" in results_full[0])
self.assertTrue("id" in results_full[0])
self.assertIn("content", results_full[0])
self.assertIn("id", results_full[0])
response = self.client.get("/api/documents/?fields=id", format="json")
self.assertEqual(response.status_code, 200)
results = response.data["results"]
self.assertFalse("content" in results[0])
self.assertTrue("id" in results[0])
self.assertIn("id", results[0])
self.assertEqual(len(results[0]), 1)
response = self.client.get("/api/documents/?fields=content", format="json")
self.assertEqual(response.status_code, 200)
results = response.data["results"]
self.assertTrue("content" in results[0])
self.assertIn("content", results[0])
self.assertFalse("id" in results[0])
self.assertEqual(len(results[0]), 1)
response = self.client.get("/api/documents/?fields=id,content", format="json")
self.assertEqual(response.status_code, 200)
results = response.data["results"]
self.assertTrue("content" in results[0])
self.assertTrue("id" in results[0])
self.assertIn("content", results[0])
self.assertIn("id", results[0])
self.assertEqual(len(results[0]), 2)
response = self.client.get(
@ -150,7 +152,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, 200)
results = response.data["results"]
self.assertFalse("content" in results[0])
self.assertTrue("id" in results[0])
self.assertIn("id", results[0])
self.assertEqual(len(results[0]), 1)
response = self.client.get("/api/documents/?fields=", format="json")
@ -505,6 +507,270 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
response = self.client.get("/api/documents/?query=content&page=3&page_size=10")
self.assertEqual(response.status_code, 404)
@override_settings(
TIME_ZONE="UTC",
)
def test_search_added_in_last_week(self):
"""
GIVEN:
- Three documents added right now
- The timezone is UTC time
WHEN:
- Query for documents added in the last 7 days
THEN:
- All three recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 week to now]")
results = response.data["results"]
# Expect 3 documents returned
self.assertEqual(len(results), 3)
for idx, subset in enumerate(
[
{"id": 1, "title": "invoice"},
{"id": 2, "title": "bank statement 1"},
{"id": 3, "title": "bank statement 3"},
],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
@override_settings(
TIME_ZONE="America/Chicago",
)
def test_search_added_in_last_week_with_timezone_behind(self):
"""
GIVEN:
- Two documents added right now
- One document added over a week ago
- The timezone is behind UTC time (-6)
WHEN:
- Query for documents added in the last 7 days
THEN:
- The two recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
# 7 days, 1 hour and 1 minute ago
added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 week to now]")
results = response.data["results"]
# Expect 2 documents returned
self.assertEqual(len(results), 2)
for idx, subset in enumerate(
[{"id": 1, "title": "invoice"}, {"id": 2, "title": "bank statement 1"}],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
@override_settings(
TIME_ZONE="Europe/Sofia",
)
def test_search_added_in_last_week_with_timezone_ahead(self):
"""
GIVEN:
- Two documents added right now
- One document added over a week ago
- The timezone is behind UTC time (+2)
WHEN:
- Query for documents added in the last 7 days
THEN:
- The two recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
# 7 days, 1 hour and 1 minute ago
added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 week to now]")
results = response.data["results"]
# Expect 2 documents returned
self.assertEqual(len(results), 2)
for idx, subset in enumerate(
[{"id": 1, "title": "invoice"}, {"id": 2, "title": "bank statement 1"}],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
def test_search_added_in_last_month(self):
"""
GIVEN:
- One document added right now
- One documents added about a week ago
- One document added over 1 month
WHEN:
- Query for documents added in the last month
THEN:
- The two recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
# 1 month, 1 day ago
added=timezone.now() - relativedelta(months=1, days=1),
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
# 7 days, 1 hour and 1 minute ago
added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 month to now]")
results = response.data["results"]
# Expect 2 documents returned
self.assertEqual(len(results), 2)
for idx, subset in enumerate(
[{"id": 1, "title": "invoice"}, {"id": 3, "title": "bank statement 3"}],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
@override_settings(
TIME_ZONE="America/Denver",
)
def test_search_added_in_last_month_timezone_behind(self):
"""
GIVEN:
- One document added right now
- One documents added about a week ago
- One document added over 1 month
- The timezone is behind UTC time (-6 or -7)
WHEN:
- Query for documents added in the last month
THEN:
- The two recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
# 1 month, 1 day ago
added=timezone.now() - relativedelta(months=1, days=1),
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
# 7 days, 1 hour and 1 minute ago
added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 month to now]")
results = response.data["results"]
# Expect 2 documents returned
self.assertEqual(len(results), 2)
for idx, subset in enumerate(
[{"id": 1, "title": "invoice"}, {"id": 3, "title": "bank statement 3"}],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
@mock.patch("documents.index.autocomplete")
def test_search_autocomplete(self, m):
m.side_effect = lambda ix, term, limit: [term for _ in range(limit)]
@ -2933,8 +3199,32 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, 400)
self.assertEqual(StoragePath.objects.count(), 1)
def test_api_storage_path_placeholders(self):
"""
GIVEN:
- API request to create a storage path with placeholders
- Storage path is valid
WHEN:
- API is called
THEN:
- Correct HTTP response
- New storage path is created
"""
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"name": "Storage path with placeholders",
"path": "{title}/{correspondent}/{document_type}/{created}/{created_year}/{created_year_short}/{created_month}/{created_month_name}/{created_month_name_short}/{created_day}/{added}/{added_year}/{added_year_short}/{added_month}/{added_month_name}/{added_month_name_short}/{added_day}/{asn}/{tags}/{tag_list}/",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, 201)
self.assertEqual(StoragePath.objects.count(), 2)
class TestTasks(APITestCase):
class TestTasks(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/tasks/"
ENDPOINT_ACKNOWLEDGE = "/api/acknowledge_tasks/"

113
src/documents/tests/test_barcodes.py
View File

@ -294,7 +294,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0])
self.assertDictEqual(separator_page_numbers, {0: False})
def test_scan_file_for_separating_barcodes_none_present(self):
"""
@ -314,7 +314,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [])
self.assertDictEqual(separator_page_numbers, {})
def test_scan_file_for_separating_barcodes_middle_page(self):
"""
@ -337,7 +337,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [1])
self.assertDictEqual(separator_page_numbers, {1: False})
def test_scan_file_for_separating_barcodes_multiple_pages(self):
"""
@ -360,7 +360,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [2, 5])
self.assertDictEqual(separator_page_numbers, {2: False, 5: False})
def test_scan_file_for_separating_barcodes_upside_down(self):
"""
@ -384,7 +384,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [1])
self.assertDictEqual(separator_page_numbers, {1: False})
def test_scan_file_for_separating_barcodes_fax_decode(self):
"""
@ -407,7 +407,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [1])
self.assertDictEqual(separator_page_numbers, {1: False})
def test_scan_file_for_separating_qr_barcodes(self):
"""
@ -431,7 +431,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0])
self.assertDictEqual(separator_page_numbers, {0: False})
@override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE")
def test_scan_file_for_separating_custom_barcodes(self):
@ -456,7 +456,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0])
self.assertDictEqual(separator_page_numbers, {0: False})
@override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE")
def test_scan_file_for_separating_custom_qr_barcodes(self):
@ -482,7 +482,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0])
self.assertDictEqual(separator_page_numbers, {0: False})
@override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE")
def test_scan_file_for_separating_custom_128_barcodes(self):
@ -508,7 +508,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0])
self.assertDictEqual(separator_page_numbers, {0: False})
def test_scan_file_for_separating_wrong_qr_barcodes(self):
"""
@ -533,7 +533,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [])
self.assertDictEqual(separator_page_numbers, {})
@override_settings(CONSUMER_BARCODE_STRING="ADAR-NEXTDOC")
def test_scan_file_for_separating_qr_barcodes(self):
@ -558,7 +558,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertGreater(len(doc_barcode_info.barcodes), 0)
self.assertListEqual(separator_page_numbers, [1])
self.assertDictEqual(separator_page_numbers, {1: False})
def test_separate_pages(self):
"""
@ -573,7 +573,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
self.BARCODE_SAMPLE_DIR,
"patch-code-t-middle.pdf",
)
documents = barcodes.separate_pages(test_file, [1])
documents = barcodes.separate_pages(test_file, {1: False})
self.assertEqual(len(documents), 2)
@ -591,7 +591,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
self.BARCODE_SAMPLE_DIR,
"patch-code-t-double.pdf",
)
pages = barcodes.separate_pages(test_file, [1, 2])
pages = barcodes.separate_pages(test_file, {1: False, 2: False})
self.assertEqual(len(pages), 2)
@ -610,7 +610,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
"patch-code-t-middle.pdf",
)
with self.assertLogs("paperless.barcodes", level="WARNING") as cm:
pages = barcodes.separate_pages(test_file, [])
pages = barcodes.separate_pages(test_file, {})
self.assertEqual(pages, [])
self.assertEqual(
cm.output,
@ -858,7 +858,88 @@ class TestBarcode(DirectoriesMixin, TestCase):
)
self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [])
self.assertDictEqual(separator_page_numbers, {})
@override_settings(
CONSUMER_ENABLE_BARCODES=True,
CONSUMER_ENABLE_ASN_BARCODE=True,
)
def test_separate_pages_by_asn_barcodes_and_patcht(self):
"""
GIVEN:
- Input PDF with a patch code on page 3 and ASN barcodes on pages 1,5,6,9,11
WHEN:
- Input file is split on barcodes
THEN:
- Correct number of files produced, split correctly by correct pages
"""
test_file = os.path.join(
os.path.dirname(__file__),
self.BARCODE_SAMPLE_DIR,
"split-by-asn-2.pdf",
)
doc_barcode_info = barcodes.scan_file_for_barcodes(
test_file,
)
separator_page_numbers = barcodes.get_separating_barcodes(
doc_barcode_info.barcodes,
)
self.assertEqual(test_file, doc_barcode_info.pdf_path)
self.assertDictEqual(
separator_page_numbers,
{
2: False,
4: True,
5: True,
8: True,
10: True,
},
)
document_list = barcodes.separate_pages(test_file, separator_page_numbers)
self.assertEqual(len(document_list), 6)
@override_settings(
CONSUMER_ENABLE_BARCODES=True,
CONSUMER_ENABLE_ASN_BARCODE=True,
)
def test_separate_pages_by_asn_barcodes(self):
"""
GIVEN:
- Input PDF with ASN barcodes on pages 1,3,4,7,9
WHEN:
- Input file is split on barcodes
THEN:
- Correct number of files produced, split correctly by correct pages
"""
test_file = os.path.join(
os.path.dirname(__file__),
self.BARCODE_SAMPLE_DIR,
"split-by-asn-1.pdf",
)
doc_barcode_info = barcodes.scan_file_for_barcodes(
test_file,
)
separator_page_numbers = barcodes.get_separating_barcodes(
doc_barcode_info.barcodes,
)
self.assertEqual(test_file, doc_barcode_info.pdf_path)
self.assertDictEqual(
separator_page_numbers,
{
2: True,
3: True,
6: True,
8: True,
},
)
document_list = barcodes.separate_pages(test_file, separator_page_numbers)
self.assertEqual(len(document_list), 5)
class TestAsnBarcodes(DirectoriesMixin, TestCase):

31
src/documents/tests/test_consumer.py
View File

@ -847,13 +847,11 @@ class PreConsumeTestCase(TestCase):
self.assertEqual(command[0], script.name)
self.assertEqual(command[1], "path-to-file")
self.assertDictContainsSubset(
{
"DOCUMENT_SOURCE_PATH": c.original_path,
"DOCUMENT_WORKING_PATH": c.path,
},
environment,
)
subset = {
"DOCUMENT_SOURCE_PATH": c.original_path,
"DOCUMENT_WORKING_PATH": c.path,
}
self.assertDictEqual(environment, {**environment, **subset})
@mock.patch("documents.consumer.Consumer.log")
def test_script_with_output(self, mocked_log):
@ -983,16 +981,15 @@ class PostConsumeTestCase(TestCase):
self.assertEqual(command[7], "my_bank")
self.assertCountEqual(command[8].split(","), ["a", "b"])
self.assertDictContainsSubset(
{
"DOCUMENT_ID": str(doc.pk),
"DOCUMENT_DOWNLOAD_URL": f"/api/documents/{doc.pk}/download/",
"DOCUMENT_THUMBNAIL_URL": f"/api/documents/{doc.pk}/thumb/",
"DOCUMENT_CORRESPONDENT": "my_bank",
"DOCUMENT_TAGS": "a,b",
},
environment,
)
subset = {
"DOCUMENT_ID": str(doc.pk),
"DOCUMENT_DOWNLOAD_URL": f"/api/documents/{doc.pk}/download/",
"DOCUMENT_THUMBNAIL_URL": f"/api/documents/{doc.pk}/thumb/",
"DOCUMENT_CORRESPONDENT": "my_bank",
"DOCUMENT_TAGS": "a,b",
}
self.assertDictEqual(environment, {**environment, **subset})
def test_script_exit_non_zero(self):
"""

7
src/documents/tests/test_importer.py
View File

@ -25,7 +25,7 @@ class TestImporter(TestCase):
cmd.manifest = [{"model": "documents.document"}]
with self.assertRaises(CommandError) as cm:
cmd._check_manifest()
self.assertTrue("The manifest file contains a record" in str(cm.exception))
self.assertIn("The manifest file contains a record", str(cm.exception))
cmd.manifest = [
{"model": "documents.document", EXPORTER_FILE_NAME: "noexist.pdf"},
@ -33,6 +33,7 @@ class TestImporter(TestCase):
# self.assertRaises(CommandError, cmd._check_manifest)
with self.assertRaises(CommandError) as cm:
cmd._check_manifest()
self.assertTrue(
'The manifest file refers to "noexist.pdf"' in str(cm.exception),
self.assertIn(
'The manifest file refers to "noexist.pdf"',
str(cm.exception),
)

59
src/documents/tests/test_index.py
View File

@ -1,3 +1,5 @@
from unittest import mock
from django.test import TestCase
from documents import index
from documents.models import Document
@ -31,3 +33,60 @@ class TestAutoComplete(DirectoriesMixin, TestCase):
)
self.assertListEqual(index.autocomplete(ix, "tes", limit=1), [b"test3"])
self.assertListEqual(index.autocomplete(ix, "tes", limit=0), [])
def test_archive_serial_number_ranging(self):
"""
GIVEN:
- Document with an archive serial number above schema allowed size
WHEN:
- Document is provided to the index
THEN:
- Error is logged
- Document ASN is reset to 0 for the index
"""
doc1 = Document.objects.create(
title="doc1",
checksum="A",
content="test test2 test3",
# yes, this is allowed, unless full_clean is run
# DRF does call the validators, this test won't
archive_serial_number=Document.ARCHIVE_SERIAL_NUMBER_MAX + 1,
)
with self.assertLogs("paperless.index", level="ERROR") as cm:
with mock.patch(
"documents.index.AsyncWriter.update_document",
) as mocked_update_doc:
index.add_or_update_document(doc1)
mocked_update_doc.assert_called_once()
_, kwargs = mocked_update_doc.call_args
self.assertEqual(kwargs["asn"], 0)
error_str = cm.output[0]
expected_str = "ERROR:paperless.index:Not indexing Archive Serial Number 4294967296 of document 1"
self.assertIn(expected_str, error_str)
def test_archive_serial_number_is_none(self):
"""
GIVEN:
- Document with no archive serial number
WHEN:
- Document is provided to the index
THEN:
- ASN isn't touched
"""
doc1 = Document.objects.create(
title="doc1",
checksum="A",
content="test test2 test3",
)
with mock.patch(
"documents.index.AsyncWriter.update_document",
) as mocked_update_doc:
index.add_or_update_document(doc1)
mocked_update_doc.assert_called_once()
_, kwargs = mocked_update_doc.call_args
self.assertIsNone(kwargs["asn"])

91
src/documents/tests/test_management_consumer.py
View File

@ -247,22 +247,85 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
def test_is_ignored(self):
test_paths = [
(os.path.join(self.dirs.consumption_dir, "foo.pdf"), False),
(os.path.join(self.dirs.consumption_dir, "foo", "bar.pdf"), False),
(os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"), True),
(
os.path.join(self.dirs.consumption_dir, "foo", ".DS_STORE", "bar.pdf"),
True,
),
(os.path.join(self.dirs.consumption_dir, ".stfolder", "foo.pdf"), True),
(os.path.join(self.dirs.consumption_dir, "._foo.pdf"), True),
(os.path.join(self.dirs.consumption_dir, "._foo", "bar.pdf"), False),
{
"path": os.path.join(self.dirs.consumption_dir, "foo.pdf"),
"ignore": False,
},
{
"path": os.path.join(self.dirs.consumption_dir, "foo", "bar.pdf"),
"ignore": False,
},
{
"path": os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"),
"ignore": True,
},
{
"path": os.path.join(
self.dirs.consumption_dir,
"foo",
".DS_STORE",
"bar.pdf",
),
"ignore": True,
},
{
"path": os.path.join(
self.dirs.consumption_dir,
".DS_STORE",
"foo",
"bar.pdf",
),
"ignore": True,
},
{
"path": os.path.join(self.dirs.consumption_dir, ".stfolder", "foo.pdf"),
"ignore": True,
},
{
"path": os.path.join(self.dirs.consumption_dir, ".stfolder.pdf"),
"ignore": False,
},
{
"path": os.path.join(
self.dirs.consumption_dir,
".stversions",
"foo.pdf",
),
"ignore": True,
},
{
"path": os.path.join(self.dirs.consumption_dir, ".stversions.pdf"),
"ignore": False,
},
{
"path": os.path.join(self.dirs.consumption_dir, "._foo.pdf"),
"ignore": True,
},
{
"path": os.path.join(self.dirs.consumption_dir, "my_foo.pdf"),
"ignore": False,
},
{
"path": os.path.join(self.dirs.consumption_dir, "._foo", "bar.pdf"),
"ignore": True,
},
{
"path": os.path.join(
self.dirs.consumption_dir,
"@eaDir",
"SYNO@.fileindexdb",
"_1jk.fnm",
),
"ignore": True,
},
]
for file_path, expected_ignored in test_paths:
for test_setup in test_paths:
filepath = test_setup["path"]
expected_ignored_result = test_setup["ignore"]
self.assertEqual(
expected_ignored,
document_consumer._is_ignored(file_path),
f'_is_ignored("{file_path}") != {expected_ignored}',
expected_ignored_result,
document_consumer._is_ignored(filepath),
f'_is_ignored("{filepath}") != {expected_ignored_result}',
)
@mock.patch("documents.management.commands.document_consumer.open")

88
src/documents/tests/test_parsers.py
View File

@ -1,6 +1,8 @@
from tempfile import TemporaryDirectory
from unittest import mock
from django.apps import apps
from django.test import override_settings
from django.test import TestCase
from documents.parsers import get_default_file_extension
from documents.parsers import get_parser_class_for_mime_type
@ -8,6 +10,7 @@ from documents.parsers import get_supported_file_extensions
from documents.parsers import is_file_ext_supported
from paperless_tesseract.parsers import RasterisedDocumentParser
from paperless_text.parsers import TextDocumentParser
from paperless_tika.parsers import TikaDocumentParser
class TestParserDiscovery(TestCase):
@ -124,14 +127,43 @@ class TestParserDiscovery(TestCase):
class TestParserAvailability(TestCase):
def test_file_extensions(self):
def test_tesseract_parser(self):
"""
GIVEN:
- Various mime types
WHEN:
- The parser class is instantiated
THEN:
- The Tesseract based parser is return
"""
supported_mimes_and_exts = [
("application/pdf", ".pdf"),
("image/png", ".png"),
("image/jpeg", ".jpg"),
("image/tiff", ".tif"),
("image/webp", ".webp"),
]
supported_exts = get_supported_file_extensions()
for mime_type, ext in supported_mimes_and_exts:
self.assertIn(ext, supported_exts)
self.assertEqual(get_default_file_extension(mime_type), ext)
self.assertIsInstance(
get_parser_class_for_mime_type(mime_type)(logging_group=None),
RasterisedDocumentParser,
)
def test_text_parser(self):
"""
GIVEN:
- Various mime types of a text form
WHEN:
- The parser class is instantiated
THEN:
- The text based parser is return
"""
supported_mimes_and_exts = [
("text/plain", ".txt"),
("text/csv", ".csv"),
]
@ -141,23 +173,55 @@ class TestParserAvailability(TestCase):
for mime_type, ext in supported_mimes_and_exts:
self.assertIn(ext, supported_exts)
self.assertEqual(get_default_file_extension(mime_type), ext)
self.assertIsInstance(
get_parser_class_for_mime_type(mime_type)(logging_group=None),
TextDocumentParser,
)
def test_tika_parser(self):
"""
GIVEN:
- Various mime types of a office document form
WHEN:
- The parser class is instantiated
THEN:
- The Tika/Gotenberg based parser is return
"""
supported_mimes_and_exts = [
("application/vnd.oasis.opendocument.text", ".odt"),
("text/rtf", ".rtf"),
("application/msword", ".doc"),
(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".docx",
),
]
# Force the app ready to notice the settings override
with override_settings(TIKA_ENABLED=True, INSTALLED_APPS=["paperless_tika"]):
app = apps.get_app_config("paperless_tika")
app.ready()
supported_exts = get_supported_file_extensions()
for mime_type, ext in supported_mimes_and_exts:
self.assertIn(ext, supported_exts)
self.assertEqual(get_default_file_extension(mime_type), ext)
self.assertIsInstance(
get_parser_class_for_mime_type(mime_type)(logging_group=None),
TikaDocumentParser,
)
def test_no_parser_for_mime(self):
self.assertIsNone(get_parser_class_for_mime_type("text/sdgsdf"))
def test_default_extension(self):
# Test no parser declared still returns a an extension
self.assertEqual(get_default_file_extension("application/zip"), ".zip")
# Test invalid mimetype returns no extension
self.assertEqual(get_default_file_extension("aasdasd/dgfgf"), "")
self.assertIsInstance(
get_parser_class_for_mime_type("application/pdf")(logging_group=None),
RasterisedDocumentParser,
)
self.assertIsInstance(
get_parser_class_for_mime_type("text/plain")(logging_group=None),
TextDocumentParser,
)
self.assertIsNone(get_parser_class_for_mime_type("text/sdgsdf"))
def test_file_extension_support(self):
self.assertTrue(is_file_ext_supported(".pdf"))
self.assertFalse(is_file_ext_supported(".hsdfh"))
self.assertFalse(is_file_ext_supported(""))

56
src/locale/nl_NL/LC_MESSAGES/django.po
View File

@ -3,7 +3,7 @@ msgstr ""
"Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2022-11-09 21:50+0000\n"
"PO-Revision-Date: 2023-01-23 12:37\n"
"PO-Revision-Date: 2023-01-27 19:22\n"
"Last-Translator: \n"
"Language-Team: Dutch\n"
"Language: nl_NL\n"
@ -368,15 +368,15 @@ msgstr "heeft tags in"
#: documents/models.py:410
msgid "ASN greater than"
msgstr ""
msgstr "ASN groter dan"
#: documents/models.py:411
msgid "ASN less than"
msgstr ""
msgstr "ASN kleiner dan"
#: documents/models.py:412
msgid "storage path is"
msgstr ""
msgstr "opslagpad is"
#: documents/models.py:422
msgid "rule type"
@ -396,99 +396,99 @@ msgstr "filterregels"
#: documents/models.py:536
msgid "Task ID"
msgstr ""
msgstr "Taak ID"
#: documents/models.py:537
msgid "Celery ID for the Task that was run"
msgstr ""
msgstr "Celery ID voor de taak die werd uitgevoerd"
#: documents/models.py:542
msgid "Acknowledged"
msgstr ""
msgstr "Bevestigd"
#: documents/models.py:543
msgid "If the task is acknowledged via the frontend or API"
msgstr ""
msgstr "Of de taak is bevestigd via de frontend of de API"
#: documents/models.py:549 documents/models.py:556
msgid "Task Name"
msgstr ""
msgstr "Taaknaam"
#: documents/models.py:550
msgid "Name of the file which the Task was run for"
msgstr ""
msgstr "Naam van het bestand waarvoor de taak werd uitgevoerd"
#: documents/models.py:557
msgid "Name of the Task which was run"
msgstr ""
msgstr "Naam van de uitgevoerde taak"
#: documents/models.py:562
msgid "Task Positional Arguments"
msgstr ""
msgstr "Positionele argumenten voor taak"
#: documents/models.py:564
msgid "JSON representation of the positional arguments used with the task"
msgstr ""
msgstr "JSON weergave van de positionele argumenten die gebruikt worden voor de taak"
#: documents/models.py:569
msgid "Task Named Arguments"
msgstr ""
msgstr "Argumenten met naam voor taak"
#: documents/models.py:571
msgid "JSON representation of the named arguments used with the task"
msgstr ""
msgstr "JSON weergave van de argumenten met naam die gebruikt worden voor de taak"
#: documents/models.py:578
msgid "Task State"
msgstr ""
msgstr "Taakstatus"
#: documents/models.py:579
msgid "Current state of the task being run"
msgstr ""
msgstr "Huidige status van de taak die wordt uitgevoerd"
#: documents/models.py:584
msgid "Created DateTime"
msgstr ""
msgstr "Aangemaakt DateTime"
#: documents/models.py:585
msgid "Datetime field when the task result was created in UTC"
msgstr ""
msgstr "Datetime veld wanneer het resultaat van de taak werd aangemaakt in UTC"
#: documents/models.py:590
msgid "Started DateTime"
msgstr ""
msgstr "Gestart DateTime"
#: documents/models.py:591
msgid "Datetime field when the task was started in UTC"
msgstr ""
msgstr "Datetime veld wanneer de taak werd gestart in UTC"
#: documents/models.py:596
msgid "Completed DateTime"
msgstr ""
msgstr "Voltooid DateTime"
#: documents/models.py:597
msgid "Datetime field when the task was completed in UTC"
msgstr ""
msgstr "Datetime veld wanneer de taak werd voltooid in UTC"
#: documents/models.py:602
msgid "Result Data"
msgstr ""
msgstr "Resultaatgegevens"
#: documents/models.py:604
msgid "The data returned by the task"
msgstr ""
msgstr "Gegevens geretourneerd door de taak"
#: documents/models.py:613
msgid "Comment for the document"
msgstr ""
msgstr "Commentaar op het document"
#: documents/models.py:642
msgid "comment"
msgstr ""
msgstr "opmerking"
#: documents/models.py:643
msgid "comments"
msgstr ""
msgstr "opmerkingen"
#: documents/serialisers.py:72
#, python-format

54
src/paperless/settings.py
View File

@ -109,6 +109,16 @@ def _parse_redis_url(env_redis: Optional[str]) -> Tuple[str]:
def _parse_beat_schedule() -> Dict:
"""
Configures the scheduled tasks, according to default or
environment variables. Task expiration is configured so the task will
expire (and not run), shortly before the default frequency will put another
of the same task into the queue
https://docs.celeryq.dev/en/stable/userguide/periodic-tasks.html#beat-entries
https://docs.celeryq.dev/en/latest/userguide/calling.html#expiration
"""
schedule = {}
tasks = [
{
@ -117,6 +127,11 @@ def _parse_beat_schedule() -> Dict:
# Default every ten minutes
"env_default": "*/10 * * * *",
"task": "paperless_mail.tasks.process_mail_accounts",
"options": {
# 1 minute before default schedule sends again
"expires": 9.0
* 60.0,
},
},
{
"name": "Train the classifier",
@ -124,6 +139,11 @@ def _parse_beat_schedule() -> Dict:
# Default hourly at 5 minutes past the hour
"env_default": "5 */1 * * *",
"task": "documents.tasks.train_classifier",
"options": {
# 1 minute before default schedule sends again
"expires": 59.0
* 60.0,
},
},
{
"name": "Optimize the index",
@ -131,6 +151,12 @@ def _parse_beat_schedule() -> Dict:
# Default daily at midnight
"env_default": "0 0 * * *",
"task": "documents.tasks.index_optimize",
"options": {
# 1 hour before default schedule sends again
"expires": 23.0
* 60.0
* 60.0,
},
},
{
"name": "Perform sanity check",
@ -138,6 +164,12 @@ def _parse_beat_schedule() -> Dict:
# Default Sunday at 00:30
"env_default": "30 0 * * sun",
"task": "documents.tasks.sanity_check",
"options": {
# 1 hour before default schedule sends again
"expires": ((7.0 * 24.0) - 1.0)
* 60.0
* 60.0,
},
},
]
for task in tasks:
@ -151,9 +183,11 @@ def _parse_beat_schedule() -> Dict:
# - five time-and-date fields
# - separated by at least one blank
minute, hour, day_month, month, day_week = value.split(" ")
schedule[task["name"]] = {
"task": task["task"],
"schedule": crontab(minute, hour, day_week, day_month, month),
"options": task["options"],
}
return schedule
@ -263,6 +297,10 @@ MIDDLEWARE = [
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
# Optional to enable compression
if __get_boolean("PAPERLESS_ENABLE_COMPRESSION", "yes"): # pragma: nocover
MIDDLEWARE.insert(0, "compression_middleware.middleware.CompressionMiddleware")
ROOT_URLCONF = "paperless.urls"
FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME")
@ -280,7 +318,6 @@ _CELERY_REDIS_URL, _CHANNELS_REDIS_URL = _parse_redis_url(
os.getenv("PAPERLESS_REDIS", None),
)
# TODO: what is this used for?
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
@ -561,22 +598,21 @@ LOGGING = {
# Task queue #
###############################################################################
TASK_WORKERS = __get_int("PAPERLESS_TASK_WORKERS", 1)
WORKER_TIMEOUT: Final[int] = __get_int("PAPERLESS_WORKER_TIMEOUT", 1800)
# https://docs.celeryq.dev/en/stable/userguide/configuration.html
CELERY_BROKER_URL = _CELERY_REDIS_URL
CELERY_TIMEZONE = TIME_ZONE
CELERY_WORKER_HIJACK_ROOT_LOGGER = False
CELERY_WORKER_CONCURRENCY = TASK_WORKERS
CELERY_WORKER_CONCURRENCY: Final[int] = __get_int("PAPERLESS_TASK_WORKERS", 1)
TASK_WORKERS = CELERY_WORKER_CONCURRENCY
CELERY_WORKER_MAX_TASKS_PER_CHILD = 1
CELERY_WORKER_SEND_TASK_EVENTS = True
CELERY_TASK_SEND_SENT_EVENT = True
CELERY_SEND_TASK_SENT_EVENT = True
CELERY_TASK_TRACK_STARTED = True
CELERY_TASK_TIME_LIMIT = WORKER_TIMEOUT
CELERY_TASK_TIME_LIMIT: Final[int] = __get_int("PAPERLESS_WORKER_TIMEOUT", 1800)
CELERY_RESULT_EXTENDED = True
CELERY_RESULT_BACKEND = "django-db"
@ -608,7 +644,7 @@ def default_threads_per_worker(task_workers) -> int:
THREADS_PER_WORKER = os.getenv(
"PAPERLESS_THREADS_PER_WORKER",
default_threads_per_worker(TASK_WORKERS),
default_threads_per_worker(CELERY_WORKER_CONCURRENCY),
)
###############################################################################
@ -637,7 +673,7 @@ CONSUMER_IGNORE_PATTERNS = list(
json.loads(
os.getenv(
"PAPERLESS_CONSUMER_IGNORE_PATTERNS",
'[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]', # noqa: E501
'[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]', # noqa: E501
),
),
)

16
src/paperless/tests/test_settings.py
View File

@ -149,6 +149,11 @@ class TestRedisSocketConversion(TestCase):
class TestCeleryScheduleParsing(TestCase):
MAIL_EXPIRE_TIME = 9.0 * 60.0
CLASSIFIER_EXPIRE_TIME = 59.0 * 60.0
INDEX_EXPIRE_TIME = 23.0 * 60.0 * 60.0
SANITY_EXPIRE_TIME = ((7.0 * 24.0) - 1.0) * 60.0 * 60.0
def test_schedule_configuration_default(self):
"""
GIVEN:
@ -165,18 +170,22 @@ class TestCeleryScheduleParsing(TestCase):
"Check all e-mail accounts": {
"task": "paperless_mail.tasks.process_mail_accounts",
"schedule": crontab(minute="*/10"),
"options": {"expires": self.MAIL_EXPIRE_TIME},
},
"Train the classifier": {
"task": "documents.tasks.train_classifier",
"schedule": crontab(minute="5", hour="*/1"),
"options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
},
"Optimize the index": {
"task": "documents.tasks.index_optimize",
"schedule": crontab(minute=0, hour=0),
"options": {"expires": self.INDEX_EXPIRE_TIME},
},
"Perform sanity check": {
"task": "documents.tasks.sanity_check",
"schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
},
},
schedule,
@ -203,18 +212,22 @@ class TestCeleryScheduleParsing(TestCase):
"Check all e-mail accounts": {
"task": "paperless_mail.tasks.process_mail_accounts",
"schedule": crontab(minute="*/50", day_of_week="mon"),
"options": {"expires": self.MAIL_EXPIRE_TIME},
},
"Train the classifier": {
"task": "documents.tasks.train_classifier",
"schedule": crontab(minute="5", hour="*/1"),
"options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
},
"Optimize the index": {
"task": "documents.tasks.index_optimize",
"schedule": crontab(minute=0, hour=0),
"options": {"expires": self.INDEX_EXPIRE_TIME},
},
"Perform sanity check": {
"task": "documents.tasks.sanity_check",
"schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
},
},
schedule,
@ -238,14 +251,17 @@ class TestCeleryScheduleParsing(TestCase):
"Check all e-mail accounts": {
"task": "paperless_mail.tasks.process_mail_accounts",
"schedule": crontab(minute="*/10"),
"options": {"expires": self.MAIL_EXPIRE_TIME},
},
"Train the classifier": {
"task": "documents.tasks.train_classifier",
"schedule": crontab(minute="5", hour="*/1"),
"options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
},
"Perform sanity check": {
"task": "documents.tasks.sanity_check",
"schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
},
},
schedule,

4
src/paperless/tests/test_websockets.py
View File

@ -14,15 +14,14 @@ TEST_CHANNEL_LAYERS = {
}
@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
class TestWebSockets(TestCase):
@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
async def test_no_auth(self):
communicator = WebsocketCommunicator(application, "/ws/status/")
connected, subprotocol = await communicator.connect()
self.assertFalse(connected)
await communicator.disconnect()
@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
@mock.patch("paperless.consumers.StatusConsumer._authenticated")
async def test_auth(self, _authenticated):
_authenticated.return_value = True
@ -33,7 +32,6 @@ class TestWebSockets(TestCase):
await communicator.disconnect()
@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
@mock.patch("paperless.consumers.StatusConsumer._authenticated")
async def test_receive(self, _authenticated):
_authenticated.return_value = True

2
src/paperless/views.py
View File

@ -12,7 +12,7 @@ class StandardPagination(PageNumberPagination):
class FaviconView(View):
def get(self, request, *args, **kwargs):
def get(self, request, *args, **kwargs): # pragma: nocover
favicon = os.path.join(
os.path.dirname(__file__),
"static",

5
src/paperless_mail/tests/test_api.py
View File

@ -2,12 +2,13 @@ from django.contrib.auth.models import User
from documents.models import Correspondent
from documents.models import DocumentType
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
from rest_framework.test import APITestCase
class TestAPIMailAccounts(APITestCase):
class TestAPIMailAccounts(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/mail_accounts/"
def setUp(self):
@ -165,7 +166,7 @@ class TestAPIMailAccounts(APITestCase):
self.assertEqual(returned_account2.password, "123xyz")
class TestAPIMailRules(APITestCase):
class TestAPIMailRules(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/mail_rules/"
def setUp(self):

15
src/paperless_mail/tests/test_parsers_live.py
View File

@ -67,11 +67,6 @@ class TestParserLive(TestCase):
return result
# Only run if convert is available
@pytest.mark.skipif(
"PAPERLESS_TEST_SKIP_CONVERT" in os.environ,
reason="PAPERLESS_TEST_SKIP_CONVERT set, skipping Test",
)
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
def test_get_thumbnail(self, mock_generate_pdf: mock.MagicMock):
"""
@ -204,11 +199,6 @@ class TestParserLive(TestCase):
"GOTENBERG_LIVE" not in os.environ,
reason="No gotenberg server",
)
# Only run if convert is available
@pytest.mark.skipif(
"PAPERLESS_TEST_SKIP_CONVERT" in os.environ,
reason="PAPERLESS_TEST_SKIP_CONVERT set, skipping Test",
)
def test_generate_pdf_from_mail(self):
"""
GIVEN:
@ -301,11 +291,6 @@ class TestParserLive(TestCase):
"GOTENBERG_LIVE" not in os.environ,
reason="No gotenberg server",
)
# Only run if convert is available
@pytest.mark.skipif(
"PAPERLESS_TEST_SKIP_CONVERT" in os.environ,
reason="PAPERLESS_TEST_SKIP_CONVERT set, skipping Test",
)
def test_generate_pdf_from_html(self):
"""
GIVEN:

2
src/paperless_tesseract/parsers.py
View File

@ -161,7 +161,7 @@ class RasterisedDocumentParser(DocumentParser):
except Exception:
# TODO catch all for various issues with PDFminer.six.
# If PDFminer fails, fall back to OCR.
# If pdftotext fails, fall back to OCR.
self.log(
"warning",
"Error while getting text from PDF document with " "pdfminer.six",

15
src/paperless_tesseract/tests/test_parser.py
View File

@ -364,7 +364,7 @@ class TestParser(DirectoriesMixin, TestCase):
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2"])
self.assertFalse("page 3" in parser.get_text().lower())
self.assertNotIn("page 3", parser.get_text().lower())
@override_settings(OCR_PAGES=1, OCR_MODE="force")
def test_multi_page_analog_pages_force(self):
@ -386,8 +386,8 @@ class TestParser(DirectoriesMixin, TestCase):
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(parser.get_text().lower(), ["page 1"])
self.assertFalse("page 2" in parser.get_text().lower())
self.assertFalse("page 3" in parser.get_text().lower())
self.assertNotIn("page 2", parser.get_text().lower())
self.assertNotIn("page 3", parser.get_text().lower())
@override_settings(OCR_MODE="skip_noarchive")
def test_skip_noarchive_withtext(self):
@ -660,6 +660,15 @@ class TestParser(DirectoriesMixin, TestCase):
params = parser.construct_ocrmypdf_parameters("", "", "", "")
self.assertNotIn("deskew", params)
with override_settings(OCR_MAX_IMAGE_PIXELS=1_000_001.0):
params = parser.construct_ocrmypdf_parameters("", "", "", "")
self.assertIn("max_image_mpixels", params)
self.assertAlmostEqual(params["max_image_mpixels"], 1, places=4)
with override_settings(OCR_MAX_IMAGE_PIXELS=-1_000_001.0):
params = parser.construct_ocrmypdf_parameters("", "", "", "")
self.assertNotIn("max_image_mpixels", params)
def test_rtl_language_detection(self):
"""
GIVEN:

2
src/paperless_tika/parsers.py
View File

@ -90,7 +90,7 @@ class TikaDocumentParser(DocumentParser):
with open(document_path, "rb") as document_handle:
files = {
"files": (
file_name or os.path.basename(document_path),
"convert" + os.path.splitext(document_path)[-1],
document_handle,
),
}

62
src/paperless_tika/tests/test_tika_parser.py
View File

@ -3,7 +3,9 @@ import os
from pathlib import Path
from unittest import mock
from django.test import override_settings
from django.test import TestCase
from documents.parsers import ParseError
from paperless_tika.parsers import TikaDocumentParser
from requests import Response
@ -54,3 +56,63 @@ class TestTikaParser(TestCase):
self.assertTrue("Creation-Date" in [m["key"] for m in metadata])
self.assertTrue("Some-key" in [m["key"] for m in metadata])
@mock.patch("paperless_tika.parsers.parser.from_file")
@mock.patch("paperless_tika.parsers.requests.post")
def test_convert_failure(self, post, from_file):
"""
GIVEN:
- Document needs to be converted to PDF
WHEN:
- Gotenberg server returns an error
THEN:
- Parse error is raised
"""
from_file.return_value = {
"content": "the content",
"metadata": {"Creation-Date": "2020-11-21"},
}
response = Response()
response._content = b"PDF document"
response.status_code = 500
post.return_value = response
file = os.path.join(self.parser.tempdir, "input.odt")
Path(file).touch()
with self.assertRaises(ParseError):
self.parser.convert_to_pdf(file, None)
@mock.patch("paperless_tika.parsers.requests.post")
def test_request_pdf_a_format(self, post: mock.Mock):
"""
GIVEN:
- Document needs to be converted to PDF
WHEN:
- Specific PDF/A format requested
THEN:
- Request to Gotenberg contains the expected PDF/A format string
"""
file = os.path.join(self.parser.tempdir, "input.odt")
Path(file).touch()
response = Response()
response._content = b"PDF document"
response.status_code = 200
post.return_value = response
for setting, expected_key in [
("pdfa", "PDF/A-2b"),
("pdfa-2", "PDF/A-2b"),
("pdfa-1", "PDF/A-1a"),
("pdfa-3", "PDF/A-3b"),
]:
with override_settings(OCR_OUTPUT_TYPE=setting):
self.parser.convert_to_pdf(file, None)
post.assert_called_once()
_, kwargs = post.call_args
self.assertEqual(kwargs["data"]["pdfFormat"], expected_key)
post.reset_mock()

2
src/setup.cfg
View File

@ -7,7 +7,7 @@ max-line-length = 88
[tool:pytest]
DJANGO_SETTINGS_MODULE=paperless.settings
addopts = --pythonwarnings=all --cov --cov-report=html --numprocesses auto --quiet
addopts = --pythonwarnings=all --cov --cov-report=html --cov-report=xml --numprocesses auto --quiet
env =
PAPERLESS_DISABLE_DBHANDLER=true