Merge branch 'dev'

This commit is contained in:
shamoon 2023-02-16 20:07:50 -08:00
commit 38de2a7767
88 changed files with 9585 additions and 21104 deletions

19
.codecov.yml Normal file
View File

@ -0,0 +1,19 @@
# https://docs.codecov.com/docs/pull-request-comments
# codecov will only comment if coverage changes
comment:
require_changes: true
coverage:
status:
project:
default:
# https://docs.codecov.com/docs/commit-status#threshold
threshold: 1%
# https://docs.codecov.com/docs/commit-status#only_pulls
only_pulls: true
patch:
default:
# For the changed lines only, target 75% covered, but
# allow as low as 50%
target: 75%
threshold: 25%
only_pulls: true

View File

@ -7,6 +7,7 @@ import subprocess
from argparse import ArgumentParser from argparse import ArgumentParser
from typing import Dict from typing import Dict
from typing import Final from typing import Final
from typing import Iterator
from typing import List from typing import List
from typing import Optional from typing import Optional
@ -15,16 +16,17 @@ from github import ContainerPackage
from github import GithubBranchApi from github import GithubBranchApi
from github import GithubContainerRegistryApi from github import GithubContainerRegistryApi
import docker
logger = logging.getLogger("cleanup-tags") logger = logging.getLogger("cleanup-tags")
class DockerManifest2: class ImageProperties:
""" """
Data class wrapping the Docker Image Manifest Version 2. Data class wrapping the properties of an entry in the image index
manifests list. It is NOT an actual image with layers, etc
See https://docs.docker.com/registry/spec/manifest-v2-2/ https://docs.docker.com/registry/spec/manifest-v2-2/
https://github.com/opencontainers/image-spec/blob/main/manifest.md
https://github.com/opencontainers/image-spec/blob/main/descriptor.md
""" """
def __init__(self, data: Dict) -> None: def __init__(self, data: Dict) -> None:
@ -41,6 +43,45 @@ class DockerManifest2:
self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}" self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
class ImageIndex:
"""
Data class wrapping up logic for an OCI Image Index
JSON data. Primary use is to access the manifests listing
See https://github.com/opencontainers/image-spec/blob/main/image-index.md
"""
def __init__(self, package_url: str, tag: str) -> None:
self.qualified_name = f"{package_url}:{tag}"
logger.info(f"Getting image index for {self.qualified_name}")
try:
proc = subprocess.run(
[
shutil.which("docker"),
"buildx",
"imagetools",
"inspect",
"--raw",
self.qualified_name,
],
capture_output=True,
check=True,
)
self._data = json.loads(proc.stdout)
except subprocess.CalledProcessError as e:
logger.error(
f"Failed to get image index for {self.qualified_name}: {e.stderr}",
)
raise e
@property
def image_pointers(self) -> Iterator[ImageProperties]:
for manifest_data in self._data["manifests"]:
yield ImageProperties(manifest_data)
class RegistryTagsCleaner: class RegistryTagsCleaner:
""" """
This is the base class for the image registry cleaning. Given a package This is the base class for the image registry cleaning. Given a package
@ -87,7 +128,10 @@ class RegistryTagsCleaner:
def clean(self): def clean(self):
""" """
This method will delete image versions, based on the selected tags to delete This method will delete image versions, based on the selected tags to delete.
It behaves more like an unlinking than actual deletion. Removing the tag
simply removes a pointer to an image, but the actual image data remains accessible
if one has the sha256 digest of it.
""" """
for tag_to_delete in self.tags_to_delete: for tag_to_delete in self.tags_to_delete:
package_version_info = self.all_pkgs_tags_to_version[tag_to_delete] package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
@ -151,31 +195,17 @@ class RegistryTagsCleaner:
# Parse manifests to locate digests pointed to # Parse manifests to locate digests pointed to
for tag in sorted(self.tags_to_keep): for tag in sorted(self.tags_to_keep):
full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
logger.info(f"Checking manifest for {full_name}")
# TODO: It would be nice to use RegistryData from docker
# except the ID doesn't map to anything in the manifest
try: try:
proc = subprocess.run( image_index = ImageIndex(
[ f"ghcr.io/{self.repo_owner}/{self.package_name}",
shutil.which("docker"), tag,
"buildx",
"imagetools",
"inspect",
"--raw",
full_name,
],
capture_output=True,
) )
for manifest in image_index.image_pointers:
manifest_list = json.loads(proc.stdout)
for manifest_data in manifest_list["manifests"]:
manifest = DockerManifest2(manifest_data)
if manifest.digest in untagged_versions: if manifest.digest in untagged_versions:
logger.info( logger.info(
f"Skipping deletion of {manifest.digest}," f"Skipping deletion of {manifest.digest},"
f" referred to by {full_name}" f" referred to by {image_index.qualified_name}"
f" for {manifest.platform}", f" for {manifest.platform}",
) )
del untagged_versions[manifest.digest] del untagged_versions[manifest.digest]
@ -247,64 +277,54 @@ class RegistryTagsCleaner:
# By default, keep anything which is tagged # By default, keep anything which is tagged
self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys())) self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
def check_tags_pull(self): def check_remaining_tags_valid(self):
""" """
This method uses the Docker Python SDK to confirm all tags which were Checks the non-deleted tags are still valid. The assumption is if the
kept still pull, for all platforms. manifest is can be inspected and each image manifest if points to can be
inspected, the image will still pull.
TODO: This is much slower (although more comprehensive). Maybe a Pool? https://github.com/opencontainers/image-spec/blob/main/image-index.md
""" """
logger.info("Beginning confirmation step") logger.info("Beginning confirmation step")
client = docker.from_env() a_tag_failed = False
imgs = []
for tag in sorted(self.tags_to_keep): for tag in sorted(self.tags_to_keep):
repository = f"ghcr.io/{self.repo_owner}/{self.package_name}"
for arch, variant in [("amd64", None), ("arm64", None), ("arm", "v7")]:
# From 11.2.0 onwards, qpdf is cross compiled, so there is a single arch, amd64
# skip others in this case
if "qpdf" in self.package_name and arch != "amd64" and tag == "11.2.0":
continue
# Skip beta and release candidate tags
elif "beta" in tag:
continue
# Build the platform name try:
if variant is not None: image_index = ImageIndex(
platform = f"linux/{arch}/{variant}" f"ghcr.io/{self.repo_owner}/{self.package_name}",
else: tag,
platform = f"linux/{arch}" )
for manifest in image_index.image_pointers:
logger.info(f"Checking {manifest.digest} for {manifest.platform}")
try: # This follows the pointer from the index to an actual image, layers and all
logger.info(f"Pulling {repository}:{tag} for {platform}") # Note the format is @
image = client.images.pull( digest_name = f"ghcr.io/{self.repo_owner}/{self.package_name}@{manifest.digest}"
repository=repository,
tag=tag,
platform=platform,
)
imgs.append(image)
except docker.errors.APIError as e:
logger.error(
f"Failed to pull {repository}:{tag}: {e}",
)
# Prevent out of space errors by removing after a few
# pulls
if len(imgs) > 50:
for image in imgs:
try: try:
client.images.remove(image.id)
except docker.errors.APIError as e: subprocess.run(
err_str = str(e) [
# Ignore attempts to remove images that are partly shared shutil.which("docker"),
# Ignore images which are somehow gone already "buildx",
if ( "imagetools",
"must be forced" not in err_str "inspect",
and "No such image" not in err_str "--raw",
): digest_name,
logger.error( ],
f"Remove image ghcr.io/{self.repo_owner}/{self.package_name}:{tag} failed: {e}", capture_output=True,
) check=True,
imgs = [] )
except subprocess.CalledProcessError as e:
logger.error(f"Failed to inspect digest: {e.stderr}")
a_tag_failed = True
except subprocess.CalledProcessError as e:
a_tag_failed = True
logger.error(f"Failed to inspect: {e.stderr}")
continue
if a_tag_failed:
raise Exception("At least one image tag failed to inspect")
class MainImageTagsCleaner(RegistryTagsCleaner): class MainImageTagsCleaner(RegistryTagsCleaner):
@ -366,7 +386,7 @@ class MainImageTagsCleaner(RegistryTagsCleaner):
class LibraryTagsCleaner(RegistryTagsCleaner): class LibraryTagsCleaner(RegistryTagsCleaner):
""" """
Exists for the off change that someday, the installer library images Exists for the off chance that someday, the installer library images
will need their own logic will need their own logic
""" """
@ -464,7 +484,7 @@ def _main():
# Verify remaining tags still pull # Verify remaining tags still pull
if args.is_manifest: if args.is_manifest:
cleaner.check_tags_pull() cleaner.check_remaining_tags_valid()
if __name__ == "__main__": if __name__ == "__main__":

View File

@ -113,16 +113,12 @@ jobs:
PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }} PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }} PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }} PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
# Skip Tests which require convert
PAPERLESS_TEST_SKIP_CONVERT: 1
# Enable Gotenberg end to end testing # Enable Gotenberg end to end testing
GOTENBERG_LIVE: 1 GOTENBERG_LIVE: 1
steps: steps:
- -
name: Checkout name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3
with:
fetch-depth: 0
- -
name: Start containers name: Start containers
run: | run: |
@ -145,6 +141,10 @@ jobs:
run: | run: |
sudo apt-get update -qq sudo apt-get update -qq
sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
-
name: Configure ImageMagick
run: |
sudo cp docker/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
- -
name: Install Python dependencies name: Install Python dependencies
run: | run: |
@ -160,27 +160,14 @@ jobs:
cd src/ cd src/
pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra
- -
name: Get changed files name: Upload coverage to Codecov
id: changed-files-specific if: ${{ matrix.python-version == env.DEFAULT_PYTHON_VERSION }}
uses: tj-actions/changed-files@v35 uses: codecov/codecov-action@v3
with: with:
files: | # not required for public repos, but intermittently fails otherwise
src/** token: ${{ secrets.CODECOV_TOKEN }}
- # future expansion
name: List all changed files flags: backend
run: |
for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
echo "${file} was changed"
done
-
name: Publish coverage results
if: matrix.python-version == ${{ env.DEFAULT_PYTHON_VERSION }} && steps.changed-files-specific.outputs.any_changed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# https://github.com/coveralls-clients/coveralls-python/issues/251
run: |
cd src/
pipenv --python ${{ steps.setup-python.outputs.python-version }} run coveralls --service=github
- -
name: Stop containers name: Stop containers
if: always() if: always()
@ -347,7 +334,7 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }} password: ${{ secrets.DOCKERHUB_TOKEN }}
- -
name: Build and push name: Build and push
uses: docker/build-push-action@v3 uses: docker/build-push-action@v4
with: with:
context: . context: .
file: ./Dockerfile file: ./Dockerfile
@ -442,21 +429,48 @@ jobs:
- -
name: Move files name: Move files
run: | run: |
mkdir dist echo "Making dist folders"
mkdir dist/paperless-ngx for directory in dist \
mkdir dist/paperless-ngx/scripts dist/paperless-ngx \
cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/ dist/paperless-ngx/scripts;
cp paperless.conf.example dist/paperless-ngx/paperless.conf do
cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py mkdir --verbose --parents ${directory}
cp -r docker/ dist/paperless-ngx/docker done
cp scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
cp -r src/ dist/paperless-ngx/src echo "Copying basic files"
cp -r docs/_build/html/ dist/paperless-ngx/docs for file_name in .dockerignore \
mv static dist/paperless-ngx .env \
Dockerfile \
Pipfile \
Pipfile.lock \
requirements.txt \
LICENSE \
README.md \
paperless.conf.example \
gunicorn.conf.py
do
cp --verbose ${file_name} dist/paperless-ngx/
done
mv --verbose dist/paperless-ngx/paperless.conf.example paperless.conf
echo "Copying Docker related files"
cp --recursive docker/ dist/paperless-ngx/docker
echo "Copying startup scripts"
cp --verbose scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
echo "Copying source files"
cp --recursive src/ dist/paperless-ngx/src
echo "Copying documentation"
cp --recursive docs/_build/html/ dist/paperless-ngx/docs
mv --verbose static dist/paperless-ngx
- -
name: Make release package name: Make release package
run: | run: |
echo "Creating release archive"
cd dist cd dist
sudo chown -R 1000:1000 paperless-ngx/
tar -cJf paperless-ngx.tar.xz paperless-ngx/ tar -cJf paperless-ngx.tar.xz paperless-ngx/
- -
name: Upload release artifact name: Upload release artifact

View File

@ -45,7 +45,7 @@ jobs:
uses: docker/setup-qemu-action@v2 uses: docker/setup-qemu-action@v2
- -
name: Build ${{ fromJSON(inputs.build-json).name }} name: Build ${{ fromJSON(inputs.build-json).name }}
uses: docker/build-push-action@v3 uses: docker/build-push-action@v4
with: with:
context: . context: .
file: ${{ inputs.dockerfile }} file: ${{ inputs.dockerfile }}

View File

@ -1 +1 @@
3.8.15 3.8.16

View File

@ -1,12 +1,12 @@
# syntax=docker/dockerfile:1.4 # syntax=docker/dockerfile:1.4
# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md
# Stage: compile-frontend
# Purpose: Compiles the frontend
# Notes:
# - Does NPM stuff with Typescript and such
FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
# This stage compiles the frontend
# This stage runs once for the native platform, as the outputs are not
# dependent on target arch
# Inputs: None
COPY ./src-ui /src/src-ui COPY ./src-ui /src/src-ui
WORKDIR /src/src-ui WORKDIR /src/src-ui
@ -16,15 +16,13 @@ RUN set -eux \
RUN set -eux \ RUN set -eux \
&& ./node_modules/.bin/ng build --configuration production && ./node_modules/.bin/ng build --configuration production
# Stage: pipenv-base
# Purpose: Generates a requirements.txt file for building
# Comments:
# - pipenv dependencies are not left in the final image
# - pipenv can't touch the final image somehow
FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base
# This stage generates the requirements.txt file using pipenv
# This stage runs once for the native platform, as the outputs are not
# dependent on target arch
# This way, pipenv dependencies are not left in the final image
# nor can pipenv mess up the final image somehow
# Inputs: None
WORKDIR /usr/src/pipenv WORKDIR /usr/src/pipenv
COPY Pipfile* ./ COPY Pipfile* ./
@ -35,6 +33,10 @@ RUN set -eux \
&& echo "Generating requirement.txt" \ && echo "Generating requirement.txt" \
&& pipenv requirements > requirements.txt && pipenv requirements > requirements.txt
# Stage: main-app
# Purpose: The final image
# Comments:
# - Don't leave anything extra in here
FROM python:3.9-slim-bullseye as main-app FROM python:3.9-slim-bullseye as main-app
LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>" LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
@ -61,10 +63,6 @@ ARG PSYCOPG2_VERSION
# Packages need for running # Packages need for running
ARG RUNTIME_PACKAGES="\ ARG RUNTIME_PACKAGES="\
# Python
python3 \
python3-pip \
python3-setuptools \
# General utils # General utils
curl \ curl \
# Docker specific # Docker specific
@ -128,7 +126,7 @@ RUN set -eux \
&& apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \ && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
&& rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/apt/lists/* \
&& echo "Installing supervisor" \ && echo "Installing supervisor" \
&& python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.4 && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.5
# Copy gunicorn config # Copy gunicorn config
# Changes very infrequently # Changes very infrequently
@ -137,7 +135,6 @@ WORKDIR /usr/src/paperless/
COPY gunicorn.conf.py . COPY gunicorn.conf.py .
# setup docker-specific things # setup docker-specific things
# Use mounts to avoid copying installer files into the image
# These change sometimes, but rarely # These change sometimes, but rarely
WORKDIR /usr/src/paperless/src/docker/ WORKDIR /usr/src/paperless/src/docker/
@ -179,7 +176,6 @@ RUN set -eux \
&& ./install_management_commands.sh && ./install_management_commands.sh
# Install the built packages from the installer library images # Install the built packages from the installer library images
# Use mounts to avoid copying installer files into the image
# These change sometimes # These change sometimes
RUN set -eux \ RUN set -eux \
&& echo "Getting binaries" \ && echo "Getting binaries" \
@ -203,7 +199,8 @@ RUN set -eux \
&& python3 -m pip list \ && python3 -m pip list \
&& echo "Cleaning up image layer" \ && echo "Cleaning up image layer" \
&& cd ../ \ && cd ../ \
&& rm -rf paperless-ngx && rm -rf paperless-ngx \
&& rm paperless-ngx.tar.gz
WORKDIR /usr/src/paperless/src/ WORKDIR /usr/src/paperless/src/
@ -247,11 +244,12 @@ COPY ./src ./
COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/ COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
# add users, setup scripts # add users, setup scripts
# Mount the compiled frontend to expected location
RUN set -eux \ RUN set -eux \
&& addgroup --gid 1000 paperless \ && addgroup --gid 1000 paperless \
&& useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \ && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
&& chown -R paperless:paperless ../ \ && chown -R paperless:paperless /usr/src/paperless \
&& gosu paperless python3 manage.py collectstatic --clear --no-input \ && gosu paperless python3 manage.py collectstatic --clear --no-input --link \
&& gosu paperless python3 manage.py compilemessages && gosu paperless python3 manage.py compilemessages
VOLUME ["/usr/src/paperless/data", \ VOLUME ["/usr/src/paperless/data", \

View File

@ -12,6 +12,7 @@ name = "piwheels"
dateparser = "~=1.1" dateparser = "~=1.1"
django = "~=4.1" django = "~=4.1"
django-cors-headers = "*" django-cors-headers = "*"
django-compression-middleware = "*"
django-extensions = "*" django-extensions = "*"
django-filter = "~=22.1" django-filter = "~=22.1"
djangorestframework = "~=3.14" djangorestframework = "~=3.14"
@ -53,21 +54,18 @@ nltk = "*"
pdf2image = "*" pdf2image = "*"
flower = "*" flower = "*"
bleach = "*" bleach = "*"
# #
# Packages locked due to issues (try to check if these are fixed in a release every so often) # Packages locked due to issues (try to check if these are fixed in a release every so often)
# #
# Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/) # Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
scipy = "==1.8.1" scipy = "==1.8.1"
# Newer versions aren't builting yet (see https://www.piwheels.org/project/cryptography/) # Newer versions aren't builting yet (see https://www.piwheels.org/project/cryptography/)
cryptography = "==38.0.1" cryptography = "==38.0.1"
# Locked version until https://github.com/django/channels_redis/issues/332 # Locked version until https://github.com/django/channels_redis/issues/332
# is resolved # is resolved
channels-redis = "==3.4.1" channels-redis = "==3.4.1"
[dev-packages] [dev-packages]
coveralls = "*" coveralls = "*"
factory-boy = "*" factory-boy = "*"

154
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "5d6da0ede3fc7dd05c9a1d836bf8786285b10b6134e763d06ee90d6e1ccb2be7" "sha256": "d70848276d3ac35fa361c15ac2d634344cdb08618790502669eee209fc16fa00"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": {}, "requires": {},
@ -117,6 +117,93 @@
"index": "pypi", "index": "pypi",
"version": "==5.0.1" "version": "==5.0.1"
}, },
"brotli": {
"hashes": [
"sha256:02177603aaca36e1fd21b091cb742bb3b305a569e2402f1ca38af471777fb019",
"sha256:11d3283d89af7033236fa4e73ec2cbe743d4f6a81d41bd234f24bf63dde979df",
"sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d",
"sha256:160c78292e98d21e73a4cc7f76a234390e516afcd982fa17e1422f7c6a9ce9c8",
"sha256:16d528a45c2e1909c2798f27f7bf0a3feec1dc9e50948e738b961618e38b6a7b",
"sha256:19598ecddd8a212aedb1ffa15763dd52a388518c4550e615aed88dc3753c0f0c",
"sha256:1c48472a6ba3b113452355b9af0a60da5c2ae60477f8feda8346f8fd48e3e87c",
"sha256:268fe94547ba25b58ebc724680609c8ee3e5a843202e9a381f6f9c5e8bdb5c70",
"sha256:269a5743a393c65db46a7bb982644c67ecba4b8d91b392403ad8a861ba6f495f",
"sha256:26d168aac4aaec9a4394221240e8a5436b5634adc3cd1cdf637f6645cecbf181",
"sha256:29d1d350178e5225397e28ea1b7aca3648fcbab546d20e7475805437bfb0a130",
"sha256:2aad0e0baa04517741c9bb5b07586c642302e5fb3e75319cb62087bd0995ab19",
"sha256:3148362937217b7072cf80a2dcc007f09bb5ecb96dae4617316638194113d5be",
"sha256:330e3f10cd01da535c70d09c4283ba2df5fb78e915bea0a28becad6e2ac010be",
"sha256:336b40348269f9b91268378de5ff44dc6fbaa2268194f85177b53463d313842a",
"sha256:3496fc835370da351d37cada4cf744039616a6db7d13c430035e901443a34daa",
"sha256:35a3edbe18e876e596553c4007a087f8bcfd538f19bc116917b3c7522fca0429",
"sha256:3b78a24b5fd13c03ee2b7b86290ed20efdc95da75a3557cc06811764d5ad1126",
"sha256:3b8b09a16a1950b9ef495a0f8b9d0a87599a9d1f179e2d4ac014b2ec831f87e7",
"sha256:3c1306004d49b84bd0c4f90457c6f57ad109f5cc6067a9664e12b7b79a9948ad",
"sha256:3ffaadcaeafe9d30a7e4e1e97ad727e4f5610b9fa2f7551998471e3736738679",
"sha256:40d15c79f42e0a2c72892bf407979febd9cf91f36f495ffb333d1d04cebb34e4",
"sha256:44bb8ff420c1d19d91d79d8c3574b8954288bdff0273bf788954064d260d7ab0",
"sha256:4688c1e42968ba52e57d8670ad2306fe92e0169c6f3af0089be75bbac0c64a3b",
"sha256:495ba7e49c2db22b046a53b469bbecea802efce200dffb69b93dd47397edc9b6",
"sha256:4d1b810aa0ed773f81dceda2cc7b403d01057458730e309856356d4ef4188438",
"sha256:503fa6af7da9f4b5780bb7e4cbe0c639b010f12be85d02c99452825dd0feef3f",
"sha256:56d027eace784738457437df7331965473f2c0da2c70e1a1f6fdbae5402e0389",
"sha256:5913a1177fc36e30fcf6dc868ce23b0453952c78c04c266d3149b3d39e1410d6",
"sha256:5b6ef7d9f9c38292df3690fe3e302b5b530999fa90014853dcd0d6902fb59f26",
"sha256:5bf37a08493232fbb0f8229f1824b366c2fc1d02d64e7e918af40acd15f3e337",
"sha256:5cb1e18167792d7d21e21365d7650b72d5081ed476123ff7b8cac7f45189c0c7",
"sha256:61a7ee1f13ab913897dac7da44a73c6d44d48a4adff42a5701e3239791c96e14",
"sha256:622a231b08899c864eb87e85f81c75e7b9ce05b001e59bbfbf43d4a71f5f32b2",
"sha256:68715970f16b6e92c574c30747c95cf8cf62804569647386ff032195dc89a430",
"sha256:6b2ae9f5f67f89aade1fab0f7fd8f2832501311c363a21579d02defa844d9296",
"sha256:6c772d6c0a79ac0f414a9f8947cc407e119b8598de7621f39cacadae3cf57d12",
"sha256:6d847b14f7ea89f6ad3c9e3901d1bc4835f6b390a9c71df999b0162d9bb1e20f",
"sha256:73fd30d4ce0ea48010564ccee1a26bfe39323fde05cb34b5863455629db61dc7",
"sha256:76ffebb907bec09ff511bb3acc077695e2c32bc2142819491579a695f77ffd4d",
"sha256:7bbff90b63328013e1e8cb50650ae0b9bac54ffb4be6104378490193cd60f85a",
"sha256:7cb81373984cc0e4682f31bc3d6be9026006d96eecd07ea49aafb06897746452",
"sha256:7ee83d3e3a024a9618e5be64648d6d11c37047ac48adff25f12fa4226cf23d1c",
"sha256:854c33dad5ba0fbd6ab69185fec8dab89e13cda6b7d191ba111987df74f38761",
"sha256:85f7912459c67eaab2fb854ed2bc1cc25772b300545fe7ed2dc03954da638649",
"sha256:87fdccbb6bb589095f413b1e05734ba492c962b4a45a13ff3408fa44ffe6479b",
"sha256:88c63a1b55f352b02c6ffd24b15ead9fc0e8bf781dbe070213039324922a2eea",
"sha256:8a674ac10e0a87b683f4fa2b6fa41090edfd686a6524bd8dedbd6138b309175c",
"sha256:8ed6a5b3d23ecc00ea02e1ed8e0ff9a08f4fc87a1f58a2530e71c0f48adf882f",
"sha256:93130612b837103e15ac3f9cbacb4613f9e348b58b3aad53721d92e57f96d46a",
"sha256:9744a863b489c79a73aba014df554b0e7a0fc44ef3f8a0ef2a52919c7d155031",
"sha256:9749a124280a0ada4187a6cfd1ffd35c350fb3af79c706589d98e088c5044267",
"sha256:97f715cf371b16ac88b8c19da00029804e20e25f30d80203417255d239f228b5",
"sha256:9bf919756d25e4114ace16a8ce91eb340eb57a08e2c6950c3cebcbe3dff2a5e7",
"sha256:9d12cf2851759b8de8ca5fde36a59c08210a97ffca0eb94c532ce7b17c6a3d1d",
"sha256:9ed4c92a0665002ff8ea852353aeb60d9141eb04109e88928026d3c8a9e5433c",
"sha256:a72661af47119a80d82fa583b554095308d6a4c356b2a554fdc2799bc19f2a43",
"sha256:afde17ae04d90fbe53afb628f7f2d4ca022797aa093e809de5c3cf276f61bbfa",
"sha256:b1375b5d17d6145c798661b67e4ae9d5496920d9265e2f00f1c2c0b5ae91fbde",
"sha256:b336c5e9cf03c7be40c47b5fd694c43c9f1358a80ba384a21969e0b4e66a9b17",
"sha256:b3523f51818e8f16599613edddb1ff924eeb4b53ab7e7197f85cbc321cdca32f",
"sha256:b43775532a5904bc938f9c15b77c613cb6ad6fb30990f3b0afaea82797a402d8",
"sha256:b663f1e02de5d0573610756398e44c130add0eb9a3fc912a09665332942a2efb",
"sha256:b83bb06a0192cccf1eb8d0a28672a1b79c74c3a8a5f2619625aeb6f28b3a82bb",
"sha256:ba72d37e2a924717990f4d7482e8ac88e2ef43fb95491eb6e0d124d77d2a150d",
"sha256:c2415d9d082152460f2bd4e382a1e85aed233abc92db5a3880da2257dc7daf7b",
"sha256:c83aa123d56f2e060644427a882a36b3c12db93727ad7a7b9efd7d7f3e9cc2c4",
"sha256:c8e521a0ce7cf690ca84b8cc2272ddaf9d8a50294fd086da67e517439614c755",
"sha256:cab1b5964b39607a66adbba01f1c12df2e55ac36c81ec6ed44f2fca44178bf1a",
"sha256:cb02ed34557afde2d2da68194d12f5719ee96cfb2eacc886352cb73e3808fc5d",
"sha256:cc0283a406774f465fb45ec7efb66857c09ffefbe49ec20b7882eff6d3c86d3a",
"sha256:cfc391f4429ee0a9370aa93d812a52e1fee0f37a81861f4fdd1f4fb28e8547c3",
"sha256:db844eb158a87ccab83e868a762ea8024ae27337fc7ddcbfcddd157f841fdfe7",
"sha256:defed7ea5f218a9f2336301e6fd379f55c655bea65ba2476346340a0ce6f74a1",
"sha256:e16eb9541f3dd1a3e92b89005e37b1257b157b7256df0e36bd7b33b50be73bcb",
"sha256:e1abbeef02962596548382e393f56e4c94acd286bd0c5afba756cffc33670e8a",
"sha256:e23281b9a08ec338469268f98f194658abfb13658ee98e2b7f85ee9dd06caa91",
"sha256:e2d9e1cbc1b25e22000328702b014227737756f4b5bf5c485ac1d8091ada078b",
"sha256:e48f4234f2469ed012a98f4b7874e7f7e173c167bed4934912a29e03167cf6b1",
"sha256:e4c4e92c14a57c9bd4cb4be678c25369bf7a092d55fd0866f759e425b9660806",
"sha256:ec1947eabbaf8e0531e8e899fc1d9876c179fc518989461f5d24e2223395a9e3",
"sha256:f909bbbc433048b499cb9db9e713b5d8d949e8c109a2a548502fb9aa8630f0b1"
],
"version": "==1.0.9"
},
"celery": { "celery": {
"extras": [ "extras": [
"redis" "redis"
@ -353,6 +440,14 @@
"index": "pypi", "index": "pypi",
"version": "==2.4.0" "version": "==2.4.0"
}, },
"django-compression-middleware": {
"hashes": [
"sha256:5e089b299a603f73254a495bf60d82eb8a7d4d636ba23dbc0ee4e4e911eee37d",
"sha256:71d4bcd09546cf88783150ac6467eb4168599534774b09b806be3017139c295b"
],
"index": "pypi",
"version": "==0.4.2"
},
"django-cors-headers": { "django-cors-headers": {
"hashes": [ "hashes": [
"sha256:37e42883b5f1f2295df6b4bba96eb2417a14a03270cb24b2a07f021cd4487cf4", "sha256:37e42883b5f1f2295df6b4bba96eb2417a14a03270cb24b2a07f021cd4487cf4",
@ -2003,6 +2098,63 @@
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==5.5.2" "version": "==5.5.2"
},
"zstandard": {
"hashes": [
"sha256:04c298d381a3b6274b0a8001f0da0ec7819d052ad9c3b0863fe8c7f154061f76",
"sha256:0fde1c56ec118940974e726c2a27e5b54e71e16c6f81d0b4722112b91d2d9009",
"sha256:126aa8433773efad0871f624339c7984a9c43913952f77d5abeee7f95a0c0860",
"sha256:1a4fb8b4ac6772e4d656103ccaf2e43e45bd16b5da324b963d58ef360d09eb73",
"sha256:2e4812720582d0803e84aefa2ac48ce1e1e6e200ca3ce1ae2be6d410c1d637ae",
"sha256:2f01b27d0b453f07cbcff01405cdd007e71f5d6410eb01303a16ba19213e58e4",
"sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863",
"sha256:3c927b6aa682c6d96225e1c797f4a5d0b9f777b327dea912b23471aaf5385376",
"sha256:3d5bb598963ac1f1f5b72dd006adb46ca6203e4fb7269a5b6e1f99e85b07ad38",
"sha256:401508efe02341ae681752a87e8ac9ef76df85ef1a238a7a21786a489d2c983d",
"sha256:4514b19abe6dbd36d6c5d75c54faca24b1ceb3999193c5b1f4b685abeabde3d0",
"sha256:47dfa52bed3097c705451bafd56dac26535545a987b6759fa39da1602349d7ba",
"sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb",
"sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980",
"sha256:55b3187e0bed004533149882ef8c24e954321f3be81f8a9ceffe35099b82a0d0",
"sha256:593f96718ad906e24d6534187fdade28b611f8ed06e27ba972ba48aecec45fc6",
"sha256:5e21032efe673b887464667d09406bab6e16d96b09ad87e80859e3a20b6745b6",
"sha256:60a86b7b2b1c300779167cf595e019e61afcc0e20c4838692983a921db9006ac",
"sha256:619f9bf37cdb4c3dc9d4120d2a1003f5db9446f3618a323219f408f6a9df6725",
"sha256:660b91eca10ee1b44c47843894abe3e6cfd80e50c90dee3123befbf7ca486bd3",
"sha256:67710d220af405f5ce22712fa741d85e8b3ada7a457ea419b038469ba379837c",
"sha256:6caed86cd47ae93915d9031dc04be5283c275e1a2af2ceff33932071f3eeff4d",
"sha256:6d2182e648e79213b3881998b30225b3f4b1f3e681f1c1eaf4cacf19bde1040d",
"sha256:72758c9f785831d9d744af282d54c3e0f9db34f7eae521c33798695464993da2",
"sha256:74c2637d12eaacb503b0b06efdf55199a11b1d7c580bd3dd9dfe84cac97ef2f6",
"sha256:755020d5aeb1b10bffd93d119e7709a2a7475b6ad79c8d5226cea3f76d152ce0",
"sha256:7ccc4727300f223184520a6064c161a90b5d0283accd72d1455bcd85ec44dd0d",
"sha256:81ab21d03e3b0351847a86a0b298b297fde1e152752614138021d6d16a476ea6",
"sha256:8371217dff635cfc0220db2720fc3ce728cd47e72bb7572cca035332823dbdfc",
"sha256:876567136b0359f6581ecd892bdb4ca03a0eead0265db73206c78cff03bcdb0f",
"sha256:879411d04068bd489db57dcf6b82ffad3c5fb2a1fdd30817c566d8b7bedee442",
"sha256:898500957ae5e7f31b7271ace4e6f3625b38c0ac84e8cedde8de3a77a7fdae5e",
"sha256:8c9ca56345b0c5574db47560603de9d05f63cce5dfeb3a456eb60f3fec737ff2",
"sha256:8ec2c146e10b59c376b6bc0369929647fcd95404a503a7aa0990f21c16462248",
"sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576",
"sha256:909bdd4e19ea437eb9b45d6695d722f6f0fd9d8f493e837d70f92062b9f39faf",
"sha256:9d97c713433087ba5cee61a3e8edb54029753d45a4288ad61a176fa4718033ce",
"sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210",
"sha256:aa9087571729c968cd853d54b3f6e9d0ec61e45cd2c31e0eb8a0d4bdbbe6da2f",
"sha256:aef0889417eda2db000d791f9739f5cecb9ccdd45c98f82c6be531bdc67ff0f2",
"sha256:b253d0c53c8ee12c3e53d181fb9ef6ce2cd9c41cbca1c56a535e4fc8ec41e241",
"sha256:b80f6f6478f9d4ca26daee6c61584499493bf97950cfaa1a02b16bb5c2c17e70",
"sha256:be6329b5ba18ec5d32dc26181e0148e423347ed936dda48bf49fb243895d1566",
"sha256:c7560f622e3849cc8f3e999791a915addd08fafe80b47fcf3ffbda5b5151047c",
"sha256:d1a7a716bb04b1c3c4a707e38e2dee46ac544fff931e66d7ae944f3019fc55b8",
"sha256:d63b04e16df8ea21dfcedbf5a60e11cbba9d835d44cb3cbff233cfd037a916d5",
"sha256:d777d239036815e9b3a093fa9208ad314c040c26d7246617e70e23025b60083a",
"sha256:e892d3177380ec080550b56a7ffeab680af25575d291766bdd875147ba246a91",
"sha256:e9c90a44470f2999779057aeaf33461cbd8bb59d8f15e983150d10bb260e16e0",
"sha256:f097dda5d4f9b9b01b3c9fa2069f9c02929365f48f341feddf3d6b32510a2f93",
"sha256:f4ebfe03cbae821ef994b2e58e4df6a087470cc522aca502614e82a143365d45"
],
"markers": "python_version >= '3.6'",
"version": "==0.19.0"
} }
}, },
"develop": { "develop": {

View File

@ -1,7 +1,7 @@
[![ci](https://github.com/paperless-ngx/paperless-ngx/workflows/ci/badge.svg)](https://github.com/paperless-ngx/paperless-ngx/actions) [![ci](https://github.com/paperless-ngx/paperless-ngx/workflows/ci/badge.svg)](https://github.com/paperless-ngx/paperless-ngx/actions)
[![Crowdin](https://badges.crowdin.net/paperless-ngx/localized.svg)](https://crowdin.com/project/paperless-ngx) [![Crowdin](https://badges.crowdin.net/paperless-ngx/localized.svg)](https://crowdin.com/project/paperless-ngx)
[![Documentation Status](https://img.shields.io/github/deployments/paperless-ngx/paperless-ngx/github-pages?label=docs)](https://docs.paperless-ngx.com) [![Documentation Status](https://img.shields.io/github/deployments/paperless-ngx/paperless-ngx/github-pages?label=docs)](https://docs.paperless-ngx.com)
[![Coverage Status](https://coveralls.io/repos/github/paperless-ngx/paperless-ngx/badge.svg?branch=master)](https://coveralls.io/github/paperless-ngx/paperless-ngx?branch=master) [![codecov](https://codecov.io/gh/paperless-ngx/paperless-ngx/branch/main/graph/badge.svg?token=VK6OUPJ3TY)](https://codecov.io/gh/paperless-ngx/paperless-ngx)
[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/%23paperlessngx%3Amatrix.org) [![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/%23paperlessngx%3Amatrix.org)
[![demo](https://cronitor.io/badges/ve7ItY/production/W5E_B9jkelG9ZbDiNHUPQEVH3MY.svg)](https://demo.paperless-ngx.com) [![demo](https://cronitor.io/badges/ve7ItY/production/W5E_B9jkelG9ZbDiNHUPQEVH3MY.svg)](https://demo.paperless-ngx.com)

View File

@ -59,7 +59,7 @@ services:
- gotenberg - gotenberg
- tika - tika
ports: ports:
- 8000:8000 - "8000:8000"
healthcheck: healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000"] test: ["CMD", "curl", "-f", "http://localhost:8000"]
interval: 30s interval: 30s

View File

@ -53,7 +53,7 @@ services:
- db - db
- broker - broker
ports: ports:
- 8000:8000 - "8000:8000"
healthcheck: healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000"] test: ["CMD", "curl", "-f", "http://localhost:8000"]
interval: 30s interval: 30s

View File

@ -53,7 +53,7 @@ services:
- db - db
- broker - broker
ports: ports:
- 8010:8000 - "8010:8000"
healthcheck: healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"] test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s interval: 30s

View File

@ -57,7 +57,7 @@ services:
- gotenberg - gotenberg
- tika - tika
ports: ports:
- 8000:8000 - "8000:8000"
healthcheck: healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"] test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s interval: 30s

View File

@ -51,7 +51,7 @@ services:
- db - db
- broker - broker
ports: ports:
- 8000:8000 - "8000:8000"
healthcheck: healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"] test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s interval: 30s

View File

@ -46,7 +46,7 @@ services:
- gotenberg - gotenberg
- tika - tika
ports: ports:
- 8000:8000 - "8000:8000"
healthcheck: healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"] test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s interval: 30s

View File

@ -37,7 +37,7 @@ services:
depends_on: depends_on:
- broker - broker
ports: ports:
- 8000:8000 - "8000:8000"
healthcheck: healthcheck:
test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"] test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
interval: 30s interval: 30s

View File

@ -80,7 +80,7 @@ django_checks() {
search_index() { search_index() {
local -r index_version=2 local -r index_version=3
local -r index_version_file=${DATA_DIR}/.index_version local -r index_version_file=${DATA_DIR}/.index_version
if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then

View File

@ -3,5 +3,10 @@
echo "Checking if we should start flower..." echo "Checking if we should start flower..."
if [[ -n "${PAPERLESS_ENABLE_FLOWER}" ]]; then if [[ -n "${PAPERLESS_ENABLE_FLOWER}" ]]; then
celery --app paperless flower # Small delay to allow celery to be up first
echo "Starting flower in 5s"
sleep 5
celery --app paperless flower --conf=/usr/src/paperless/src/paperless/flowerconfig.py
else
echo "Not starting flower"
fi fi

View File

@ -141,7 +141,8 @@ directory.
files created using "collectstatic" manager command are stored. files created using "collectstatic" manager command are stored.
Unless you're doing something fancy, there is no need to override Unless you're doing something fancy, there is no need to override
this. this. If this is changed, you may need to run
`collectstatic` again.
Defaults to "../static/", relative to the "src" directory. Defaults to "../static/", relative to the "src" directory.
@ -757,6 +758,18 @@ should be a valid crontab(5) expression describing when to run.
Defaults to `30 0 * * sun` or Sunday at 30 minutes past midnight. Defaults to `30 0 * * sun` or Sunday at 30 minutes past midnight.
`PAPERLESS_ENABLE_COMPRESSION=<bool>`
: Enables compression of the responses from the webserver.
: Defaults to 1, enabling compression.
!!! note
If you are using a proxy such as nginx, it is likely more efficient
to enable compression in your proxy configuration rather than
the webserver
## Polling {#polling} ## Polling {#polling}
`PAPERLESS_CONSUMER_POLLING=<num>` `PAPERLESS_CONSUMER_POLLING=<num>`
@ -986,13 +999,20 @@ within your documents.
`PAPERLESS_CONSUMER_IGNORE_PATTERNS=<json>` `PAPERLESS_CONSUMER_IGNORE_PATTERNS=<json>`
: By default, paperless ignores certain files and folders in the : By default, paperless ignores certain files and folders in the
consumption directory, such as system files created by the Mac OS. consumption directory, such as system files created by the Mac OS
or hidden folders some tools use to store data.
This can be adjusted by configuring a custom json array with This can be adjusted by configuring a custom json array with
patterns to exclude. patterns to exclude.
For example, `.DS_STORE/*` will ignore any files found in a folder
named `.DS_STORE`, including `.DS_STORE/bar.pdf` and `foo/.DS_STORE/bar.pdf`
A pattern like `._*` will ignore anything starting with `._`, including:
`._foo.pdf` and `._bar/foo.pdf`
Defaults to Defaults to
`[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]`. `[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]`.
## Binaries ## Binaries

View File

@ -321,7 +321,7 @@ fi
wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/docker-compose.$DOCKER_COMPOSE_VERSION.yml" -O docker-compose.yml wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/docker-compose.$DOCKER_COMPOSE_VERSION.yml" -O docker-compose.yml
wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/.env" -O .env wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/.env" -O .env
SECRET_KEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 64 | head -n 1) SECRET_KEY=$(tr --delete --complement 'a-zA-Z0-9' < /dev/urandom 2>/dev/null | head --bytes 64)
DEFAULT_LANGUAGES=("deu eng fra ita spa") DEFAULT_LANGUAGES=("deu eng fra ita spa")
@ -346,7 +346,7 @@ read -r -a OCR_LANGUAGES_ARRAY <<< "${_split_langs}"
fi fi
} > docker-compose.env } > docker-compose.env
sed -i "s/- 8000:8000/- $PORT:8000/g" docker-compose.yml sed -i "s/- \"8000:8000\"/- \"$PORT:8000\"/g" docker-compose.yml
sed -i "s#- \./consume:/usr/src/paperless/consume#- $CONSUME_FOLDER:/usr/src/paperless/consume#g" docker-compose.yml sed -i "s#- \./consume:/usr/src/paperless/consume#- $CONSUME_FOLDER:/usr/src/paperless/consume#g" docker-compose.yml

View File

@ -192,7 +192,8 @@
"cli": { "cli": {
"schematicCollections": [ "schematicCollections": [
"@angular-eslint/schematics" "@angular-eslint/schematics"
] ],
"analytics": false
}, },
"schematics": { "schematics": {
"@angular-eslint/schematics:application": { "@angular-eslint/schematics:application": {

View File

@ -445,6 +445,10 @@
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">1</context> <context context-type="linenumber">1</context>
</context-group> </context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">192</context>
</context-group>
</trans-unit> </trans-unit>
<trans-unit id="3797778920049399855" datatype="html"> <trans-unit id="3797778920049399855" datatype="html">
<source>Logout</source> <source>Logout</source>
@ -711,7 +715,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">492</context> <context context-type="linenumber">498</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2526035785704676448" datatype="html"> <trans-unit id="2526035785704676448" datatype="html">
@ -879,15 +883,15 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">196</context> <context context-type="linenumber">204</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">248</context> <context context-type="linenumber">256</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">283</context> <context context-type="linenumber">291</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context> <context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context>
@ -1026,7 +1030,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">317</context> <context context-type="linenumber">325</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6457471243969293847" datatype="html"> <trans-unit id="6457471243969293847" datatype="html">
@ -1163,7 +1167,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">284</context> <context context-type="linenumber">292</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7046259383943324039" datatype="html"> <trans-unit id="7046259383943324039" datatype="html">
@ -1401,15 +1405,15 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">214</context> <context context-type="linenumber">222</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">261</context> <context context-type="linenumber">269</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">296</context> <context context-type="linenumber">304</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2784260611081866636" datatype="html"> <trans-unit id="2784260611081866636" datatype="html">
@ -1703,11 +1707,11 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">222</context> <context context-type="linenumber">230</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">308</context> <context context-type="linenumber">316</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context> <context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context>
@ -2031,7 +2035,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context> <context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">32</context> <context context-type="linenumber">34</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="4452427314943113135" datatype="html"> <trans-unit id="4452427314943113135" datatype="html">
@ -2269,7 +2273,7 @@
<source>Confirm delete</source> <source>Confirm delete</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">449</context> <context context-type="linenumber">453</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/management-list/management-list.component.ts</context> <context context-type="sourcefile">src/app/components/manage/management-list/management-list.component.ts</context>
@ -2280,35 +2284,35 @@
<source>Do you really want to delete document &quot;<x id="PH" equiv-text="this.document.title"/>&quot;?</source> <source>Do you really want to delete document &quot;<x id="PH" equiv-text="this.document.title"/>&quot;?</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">450</context> <context context-type="linenumber">454</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6691075929777935948" datatype="html"> <trans-unit id="6691075929777935948" datatype="html">
<source>The files for this document will be deleted permanently. This operation cannot be undone.</source> <source>The files for this document will be deleted permanently. This operation cannot be undone.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">451</context> <context context-type="linenumber">455</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="719892092227206532" datatype="html"> <trans-unit id="719892092227206532" datatype="html">
<source>Delete document</source> <source>Delete document</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">453</context> <context context-type="linenumber">457</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1844801255494293730" datatype="html"> <trans-unit id="1844801255494293730" datatype="html">
<source>Error deleting document: <x id="PH" equiv-text="JSON.stringify(error)"/></source> <source>Error deleting document: <x id="PH" equiv-text="JSON.stringify(error)"/></source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">469</context> <context context-type="linenumber">473</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7362691899087997122" datatype="html"> <trans-unit id="7362691899087997122" datatype="html">
<source>Redo OCR confirm</source> <source>Redo OCR confirm</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">489</context> <context context-type="linenumber">493</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context> <context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@ -2319,14 +2323,14 @@
<source>This operation will permanently redo OCR for this document.</source> <source>This operation will permanently redo OCR for this document.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">490</context> <context context-type="linenumber">494</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5641451190833696892" datatype="html"> <trans-unit id="5641451190833696892" datatype="html">
<source>This operation cannot be undone.</source> <source>This operation cannot be undone.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">491</context> <context context-type="linenumber">495</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context> <context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@ -2338,18 +2342,18 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">576</context> <context context-type="linenumber">582</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">635</context> <context context-type="linenumber">641</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1181910457994920507" datatype="html"> <trans-unit id="1181910457994920507" datatype="html">
<source>Proceed</source> <source>Proceed</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">493</context> <context context-type="linenumber">497</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context> <context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@ -2357,18 +2361,18 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">578</context> <context context-type="linenumber">584</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">637</context> <context context-type="linenumber">643</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5729001209753056399" datatype="html"> <trans-unit id="5729001209753056399" datatype="html">
<source>Redo OCR operation will begin in the background. Close and re-open or reload this document after the operation has completed to see new content.</source> <source>Redo OCR operation will begin in the background. Close and re-open or reload this document after the operation has completed to see new content.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">501</context> <context context-type="linenumber">505</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="8008978164775353960" datatype="html"> <trans-unit id="8008978164775353960" datatype="html">
@ -2377,7 +2381,7 @@
)"/></source> )"/></source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">512,514</context> <context context-type="linenumber">516,518</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6857598786757174736" datatype="html"> <trans-unit id="6857598786757174736" datatype="html">
@ -2462,15 +2466,15 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">213</context> <context context-type="linenumber">221</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">250</context> <context context-type="linenumber">258</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">285</context> <context context-type="linenumber">293</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context> <context context-type="sourcefile">src/app/components/manage/tasks/tasks.component.html</context>
@ -2748,11 +2752,11 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">260</context> <context context-type="linenumber">268</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">295</context> <context context-type="linenumber">303</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2509141182388535183" datatype="html"> <trans-unit id="2509141182388535183" datatype="html">
@ -2886,6 +2890,10 @@
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context> <context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
<context context-type="linenumber">64</context> <context context-type="linenumber">64</context>
</context-group> </context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">199</context>
</context-group>
</trans-unit> </trans-unit>
<trans-unit id="1233494216161906927" datatype="html"> <trans-unit id="1233494216161906927" datatype="html">
<source>Save &quot;<x id="INTERPOLATION" equiv-text="{{list.activeSavedViewTitle}}"/>&quot;</source> <source>Save &quot;<x id="INTERPOLATION" equiv-text="{{list.activeSavedViewTitle}}"/>&quot;</source>
@ -3114,7 +3122,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">208</context> <context context-type="linenumber">216</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="4104807402967139762" datatype="html"> <trans-unit id="4104807402967139762" datatype="html">
@ -3125,7 +3133,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">204</context> <context context-type="linenumber">212</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6965614903949668392" datatype="html"> <trans-unit id="6965614903949668392" datatype="html">
@ -3647,123 +3655,130 @@
<context context-type="linenumber">181</context> <context context-type="linenumber">181</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1595668988802980095" datatype="html">
<source>Show warning when closing saved views with unsaved changes</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">195</context>
</context-group>
</trans-unit>
<trans-unit id="6925788033494878061" datatype="html"> <trans-unit id="6925788033494878061" datatype="html">
<source>Appears on</source> <source>Appears on</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">201</context> <context context-type="linenumber">209</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7877440816920439876" datatype="html"> <trans-unit id="7877440816920439876" datatype="html">
<source>No saved views defined.</source> <source>No saved views defined.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">218</context> <context context-type="linenumber">226</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1292737233370901804" datatype="html"> <trans-unit id="1292737233370901804" datatype="html">
<source>Mail</source> <source>Mail</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">232,231</context> <context context-type="linenumber">240,239</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="8913167930428886792" datatype="html"> <trans-unit id="8913167930428886792" datatype="html">
<source>Mail accounts</source> <source>Mail accounts</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">236</context> <context context-type="linenumber">244</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1259421956660976189" datatype="html"> <trans-unit id="1259421956660976189" datatype="html">
<source>Add Account</source> <source>Add Account</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">241</context> <context context-type="linenumber">249</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2188854519574316630" datatype="html"> <trans-unit id="2188854519574316630" datatype="html">
<source>Server</source> <source>Server</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">249</context> <context context-type="linenumber">257</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6235247415162820954" datatype="html"> <trans-unit id="6235247415162820954" datatype="html">
<source>No mail accounts defined.</source> <source>No mail accounts defined.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">267</context> <context context-type="linenumber">275</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5364020217520256833" datatype="html"> <trans-unit id="5364020217520256833" datatype="html">
<source>Mail rules</source> <source>Mail rules</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">271</context> <context context-type="linenumber">279</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1372022816709469401" datatype="html"> <trans-unit id="1372022816709469401" datatype="html">
<source>Add Rule</source> <source>Add Rule</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">276</context> <context context-type="linenumber">284</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6751234988479444294" datatype="html"> <trans-unit id="6751234988479444294" datatype="html">
<source>No mail rules defined.</source> <source>No mail rules defined.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
<context context-type="linenumber">302</context> <context context-type="linenumber">310</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5610279464668232148" datatype="html"> <trans-unit id="5610279464668232148" datatype="html">
<source>Saved view &quot;<x id="PH" equiv-text="savedView.name"/>&quot; deleted.</source> <source>Saved view &quot;<x id="PH" equiv-text="savedView.name"/>&quot; deleted.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">383</context> <context context-type="linenumber">385</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="3891152409365583719" datatype="html"> <trans-unit id="3891152409365583719" datatype="html">
<source>Settings saved</source> <source>Settings saved</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">476</context> <context context-type="linenumber">482</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7217000812750597833" datatype="html"> <trans-unit id="7217000812750597833" datatype="html">
<source>Settings were saved successfully.</source> <source>Settings were saved successfully.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">477</context> <context context-type="linenumber">483</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="525012668859298131" datatype="html"> <trans-unit id="525012668859298131" datatype="html">
<source>Settings were saved successfully. Reload is required to apply some changes.</source> <source>Settings were saved successfully. Reload is required to apply some changes.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">481</context> <context context-type="linenumber">487</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="8491974984518503778" datatype="html"> <trans-unit id="8491974984518503778" datatype="html">
<source>Reload now</source> <source>Reload now</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">482</context> <context context-type="linenumber">488</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6839066544204061364" datatype="html"> <trans-unit id="6839066544204061364" datatype="html">
<source>Use system language</source> <source>Use system language</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">500</context> <context context-type="linenumber">506</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7729897675462249787" datatype="html"> <trans-unit id="7729897675462249787" datatype="html">
<source>Use date format of display language</source> <source>Use date format of display language</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">507</context> <context context-type="linenumber">513</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="8488620293789898901" datatype="html"> <trans-unit id="8488620293789898901" datatype="html">
@ -3772,91 +3787,91 @@
)"/></source> )"/></source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">527,529</context> <context context-type="linenumber">533,535</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6327501535846658797" datatype="html"> <trans-unit id="6327501535846658797" datatype="html">
<source>Saved account &quot;<x id="PH" equiv-text="newMailAccount.name"/>&quot;.</source> <source>Saved account &quot;<x id="PH" equiv-text="newMailAccount.name"/>&quot;.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">554</context> <context context-type="linenumber">560</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6428427497555765743" datatype="html"> <trans-unit id="6428427497555765743" datatype="html">
<source>Error saving account: <x id="PH" equiv-text="e.toString()"/>.</source> <source>Error saving account: <x id="PH" equiv-text="e.toString()"/>.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">564</context> <context context-type="linenumber">570</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5641934153807844674" datatype="html"> <trans-unit id="5641934153807844674" datatype="html">
<source>Confirm delete mail account</source> <source>Confirm delete mail account</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">574</context> <context context-type="linenumber">580</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7176985344323395435" datatype="html"> <trans-unit id="7176985344323395435" datatype="html">
<source>This operation will permanently delete this mail account.</source> <source>This operation will permanently delete this mail account.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">575</context> <context context-type="linenumber">581</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="4233826387148482123" datatype="html"> <trans-unit id="4233826387148482123" datatype="html">
<source>Deleted mail account</source> <source>Deleted mail account</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">584</context> <context context-type="linenumber">590</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7443801450153832973" datatype="html"> <trans-unit id="7443801450153832973" datatype="html">
<source>Error deleting mail account: <x id="PH" equiv-text="e.toString()"/>.</source> <source>Error deleting mail account: <x id="PH" equiv-text="e.toString()"/>.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">593</context> <context context-type="linenumber">599</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="123368655395433699" datatype="html"> <trans-unit id="123368655395433699" datatype="html">
<source>Saved rule &quot;<x id="PH" equiv-text="newMailRule.name"/>&quot;.</source> <source>Saved rule &quot;<x id="PH" equiv-text="newMailRule.name"/>&quot;.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">612</context> <context context-type="linenumber">618</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="4741216051394823471" datatype="html"> <trans-unit id="4741216051394823471" datatype="html">
<source>Error saving rule: <x id="PH" equiv-text="e.toString()"/>.</source> <source>Error saving rule: <x id="PH" equiv-text="e.toString()"/>.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">623</context> <context context-type="linenumber">629</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="3896080636020672118" datatype="html"> <trans-unit id="3896080636020672118" datatype="html">
<source>Confirm delete mail rule</source> <source>Confirm delete mail rule</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">633</context> <context context-type="linenumber">639</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2250372580580310337" datatype="html"> <trans-unit id="2250372580580310337" datatype="html">
<source>This operation will permanently delete this mail rule.</source> <source>This operation will permanently delete this mail rule.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">634</context> <context context-type="linenumber">640</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="9077981247971516916" datatype="html"> <trans-unit id="9077981247971516916" datatype="html">
<source>Deleted mail rule</source> <source>Deleted mail rule</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">643</context> <context context-type="linenumber">649</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="4740074357089345173" datatype="html"> <trans-unit id="4740074357089345173" datatype="html">
<source>Error deleting mail rule: <x id="PH" equiv-text="e.toString()"/>.</source> <source>Error deleting mail rule: <x id="PH" equiv-text="e.toString()"/>.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context> <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
<context context-type="linenumber">652</context> <context context-type="linenumber">658</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5101757640976222639" datatype="html"> <trans-unit id="5101757640976222639" datatype="html">
@ -4113,7 +4128,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context> <context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">24</context> <context context-type="linenumber">26</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/services/open-documents.service.ts</context> <context context-type="sourcefile">src/app/services/open-documents.service.ts</context>
@ -4153,21 +4168,21 @@
<source>You have unsaved changes to the saved view</source> <source>You have unsaved changes to the saved view</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context> <context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">26</context> <context context-type="linenumber">28</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7282050913165342352" datatype="html"> <trans-unit id="7282050913165342352" datatype="html">
<source>Are you sure you want to close this saved view?</source> <source>Are you sure you want to close this saved view?</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context> <context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">30</context> <context context-type="linenumber">32</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="856284624775342512" datatype="html"> <trans-unit id="856284624775342512" datatype="html">
<source>Save and close</source> <source>Save and close</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context> <context context-type="sourcefile">src/app/guards/dirty-saved-view.guard.ts</context>
<context context-type="linenumber">34</context> <context context-type="linenumber">36</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7536524521722799066" datatype="html"> <trans-unit id="7536524521722799066" datatype="html">

14779
src-ui/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -13,14 +13,14 @@
}, },
"private": true, "private": true,
"dependencies": { "dependencies": {
"@angular/common": "~15.1.0", "@angular/common": "~15.1.2",
"@angular/compiler": "~15.1.0", "@angular/compiler": "~15.1.2",
"@angular/core": "~15.1.0", "@angular/core": "~15.1.2",
"@angular/forms": "~15.1.0", "@angular/forms": "~15.1.2",
"@angular/localize": "~15.1.0", "@angular/localize": "~15.1.2",
"@angular/platform-browser": "~15.1.0", "@angular/platform-browser": "~15.1.2",
"@angular/platform-browser-dynamic": "~15.1.0", "@angular/platform-browser-dynamic": "~15.1.2",
"@angular/router": "~15.1.0", "@angular/router": "~15.1.2",
"@ng-bootstrap/ng-bootstrap": "^14.0.1", "@ng-bootstrap/ng-bootstrap": "^14.0.1",
"@ng-select/ng-select": "^10.0.1", "@ng-select/ng-select": "^10.0.1",
"@ngneat/dirty-check-forms": "^3.0.3", "@ngneat/dirty-check-forms": "^3.0.3",
@ -39,18 +39,18 @@
}, },
"devDependencies": { "devDependencies": {
"@angular-builders/jest": "15.0.0", "@angular-builders/jest": "15.0.0",
"@angular-devkit/build-angular": "~15.1.0", "@angular-devkit/build-angular": "~15.1.4",
"@angular-eslint/builder": "15.1.0", "@angular-eslint/builder": "15.2.0",
"@angular-eslint/eslint-plugin": "15.1.0", "@angular-eslint/eslint-plugin": "15.2.0",
"@angular-eslint/eslint-plugin-template": "15.1.0", "@angular-eslint/eslint-plugin-template": "15.2.0",
"@angular-eslint/schematics": "15.1.0", "@angular-eslint/schematics": "15.2.0",
"@angular-eslint/template-parser": "15.1.0", "@angular-eslint/template-parser": "15.2.0",
"@angular/cli": "~15.1.0", "@angular/cli": "~15.1.4",
"@angular/compiler-cli": "~15.1.0", "@angular/compiler-cli": "~15.1.2",
"@types/jest": "28.1.6", "@types/jest": "28.1.6",
"@types/node": "^18.7.23", "@types/node": "^18.7.23",
"@typescript-eslint/eslint-plugin": "^5.43.0", "@typescript-eslint/eslint-plugin": "^5.43.0",
"@typescript-eslint/parser": "^5.43.0", "@typescript-eslint/parser": "^5.50.0",
"concurrently": "7.4.0", "concurrently": "7.4.0",
"eslint": "^8.31.0", "eslint": "^8.31.0",
"jest": "28.1.3", "jest": "28.1.3",

View File

@ -17,7 +17,7 @@
(blur)="onBlur()"> (blur)="onBlur()">
<ng-template ng-label-tmp let-item="item"> <ng-template ng-label-tmp let-item="item">
<span class="tag-wrap tag-wrap-delete" (click)="removeTag(item.id)"> <span class="tag-wrap tag-wrap-delete" (mousedown)="removeTag($event, item.id)">
<svg width="1.2em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg"> <svg width="1.2em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
<use xlink:href="assets/bootstrap-icons.svg#x"/> <use xlink:href="assets/bootstrap-icons.svg#x"/>
</svg> </svg>

View File

@ -65,7 +65,7 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
private _lastSearchTerm: string private _lastSearchTerm: string
getTag(id) { getTag(id: number) {
if (this.tags) { if (this.tags) {
return this.tags.find((tag) => tag.id == id) return this.tags.find((tag) => tag.id == id)
} else { } else {
@ -73,7 +73,10 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
} }
} }
removeTag(id) { removeTag(event: PointerEvent, id: number) {
// prevent opening dropdown
event.stopImmediatePropagation()
let index = this.value.indexOf(id) let index = this.value.indexOf(id)
if (index > -1) { if (index > -1) {
let oldValue = this.value let oldValue = this.value

View File

@ -63,7 +63,7 @@
<div class="row"> <div class="row">
<div class="col mb-4"> <div class="col-md-6 col-xl-4 mb-4">
<form [formGroup]='documentForm' (ngSubmit)="save()"> <form [formGroup]='documentForm' (ngSubmit)="save()">

View File

@ -22,6 +22,15 @@
--page-margin: 1px 0 20px; --page-margin: 1px 0 20px;
} }
::ng-deep .ng-select-taggable {
max-width: calc(100% - 46px); // fudge factor for ng-select button width
}
.btn-group .dropdown-toggle-split {
border-top-right-radius: inherit;
border-bottom-right-radius: inherit;
}
.password-prompt { .password-prompt {
position: absolute; position: absolute;
top: 30%; top: 30%;

View File

@ -10,7 +10,7 @@
</div> </div>
</div> </div>
<ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination> <ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" [maxSize]="5" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
</div> </div>
<table class="table table-striped align-middle border shadow-sm"> <table class="table table-striped align-middle border shadow-sm">
@ -72,5 +72,5 @@
<div class="d-flex"> <div class="d-flex">
<div i18n *ngIf="collectionSize > 0">{collectionSize, plural, =1 {One {{typeName}}} other {{{collectionSize || 0}} total {{typeNamePlural}}}}</div> <div i18n *ngIf="collectionSize > 0">{collectionSize, plural, =1 {One {{typeName}}} other {{{collectionSize || 0}} total {{typeNamePlural}}}}</div>
<ngb-pagination *ngIf="collectionSize > 20" class="ms-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination> <ngb-pagination *ngIf="collectionSize > 20" class="ms-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" [maxSize]="5" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
</div> </div>

View File

@ -189,6 +189,14 @@
<a ngbNavLink i18n>Saved views</a> <a ngbNavLink i18n>Saved views</a>
<ng-template ngbNavContent> <ng-template ngbNavContent>
<h4 i18n>Settings</h4>
<div class="row mb-3">
<div class="offset-md-3 col">
<app-input-check i18n-title title="Show warning when closing saved views with unsaved changes" formControlName="savedViewsWarnOnUnsavedChange"></app-input-check>
</div>
</div>
<h4 i18n>Views</h4>
<div formGroupName="savedViews"> <div formGroupName="savedViews">
<div *ngFor="let view of savedViews" [formGroupName]="view.id" class="row"> <div *ngFor="let view of savedViews" [formGroupName]="view.id" class="row">

View File

@ -84,6 +84,7 @@ export class SettingsComponent
notificationsConsumerFailed: new FormControl(null), notificationsConsumerFailed: new FormControl(null),
notificationsConsumerSuppressOnDashboard: new FormControl(null), notificationsConsumerSuppressOnDashboard: new FormControl(null),
savedViewsWarnOnUnsavedChange: new FormControl(null),
savedViews: this.savedViewGroup, savedViews: this.savedViewGroup,
mailAccounts: this.mailAccountGroup, mailAccounts: this.mailAccountGroup,
@ -194,6 +195,9 @@ export class SettingsComponent
notificationsConsumerSuppressOnDashboard: this.settings.get( notificationsConsumerSuppressOnDashboard: this.settings.get(
SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD
), ),
savedViewsWarnOnUnsavedChange: this.settings.get(
SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE
),
savedViews: {}, savedViews: {},
mailAccounts: {}, mailAccounts: {},
mailRules: {}, mailRules: {},
@ -462,6 +466,10 @@ export class SettingsComponent
SETTINGS_KEYS.UPDATE_CHECKING_ENABLED, SETTINGS_KEYS.UPDATE_CHECKING_ENABLED,
this.settingsForm.value.updateCheckingEnabled this.settingsForm.value.updateCheckingEnabled
) )
this.settings.set(
SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE,
this.settingsForm.value.savedViewsWarnOnUnsavedChange
)
this.settings.setLanguage(this.settingsForm.value.displayLanguage) this.settings.setLanguage(this.settingsForm.value.displayLanguage)
this.settings this.settings
.storeSettings() .storeSettings()

View File

@ -41,6 +41,8 @@ export const SETTINGS_KEYS = {
UPDATE_CHECKING_ENABLED: 'general-settings:update-checking:enabled', UPDATE_CHECKING_ENABLED: 'general-settings:update-checking:enabled',
UPDATE_CHECKING_BACKEND_SETTING: UPDATE_CHECKING_BACKEND_SETTING:
'general-settings:update-checking:backend-setting', 'general-settings:update-checking:backend-setting',
SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE:
'general-settings:saved-views:warn-on-unsaved-change',
} }
export const SETTINGS: PaperlessUiSetting[] = [ export const SETTINGS: PaperlessUiSetting[] = [
@ -139,4 +141,9 @@ export const SETTINGS: PaperlessUiSetting[] = [
type: 'string', type: 'string',
default: '', default: '',
}, },
{
key: SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE,
type: 'boolean',
default: true,
},
] ]

View File

@ -4,17 +4,25 @@ import { first, Observable, Subject } from 'rxjs'
import { DocumentListComponent } from '../components/document-list/document-list.component' import { DocumentListComponent } from '../components/document-list/document-list.component'
import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
import { ConfirmDialogComponent } from '../components/common/confirm-dialog/confirm-dialog.component' import { ConfirmDialogComponent } from '../components/common/confirm-dialog/confirm-dialog.component'
import { SettingsService } from '../services/settings.service'
import { SETTINGS_KEYS } from '../data/paperless-uisettings'
@Injectable() @Injectable()
export class DirtySavedViewGuard export class DirtySavedViewGuard
implements CanDeactivate<DocumentListComponent> implements CanDeactivate<DocumentListComponent>
{ {
constructor(private modalService: NgbModal) {} constructor(
private modalService: NgbModal,
private settings: SettingsService
) {}
canDeactivate( canDeactivate(
component: DocumentListComponent component: DocumentListComponent
): boolean | Observable<boolean> { ): boolean | Observable<boolean> {
return component.savedViewIsModified ? this.warn(component) : true return component.savedViewIsModified &&
this.settings.get(SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE)
? this.warn(component)
: true
} }
warn(component: DocumentListComponent) { warn(component: DocumentListComponent) {

View File

@ -5,7 +5,7 @@ export const environment = {
apiBaseUrl: document.baseURI + 'api/', apiBaseUrl: document.baseURI + 'api/',
apiVersion: '2', apiVersion: '2',
appTitle: 'Paperless-ngx', appTitle: 'Paperless-ngx',
version: '1.12.2', version: '1.12.2-dev',
webSocketHost: window.location.host, webSocketHost: window.location.host,
webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:', webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
webSocketBaseUrl: base_url.pathname + 'ws/', webSocketBaseUrl: base_url.pathname + 'ws/',

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -5,6 +5,7 @@ import tempfile
from dataclasses import dataclass from dataclasses import dataclass
from functools import lru_cache from functools import lru_cache
from pathlib import Path from pathlib import Path
from typing import Dict
from typing import List from typing import List
from typing import Optional from typing import Optional
@ -201,16 +202,25 @@ def scan_file_for_barcodes(
return DocumentBarcodeInfo(pdf_filepath, barcodes) return DocumentBarcodeInfo(pdf_filepath, barcodes)
def get_separating_barcodes(barcodes: List[Barcode]) -> List[int]: def get_separating_barcodes(barcodes: List[Barcode]) -> Dict[int, bool]:
""" """
Search the parsed barcodes for separators Search the parsed barcodes for separators
and returns a list of page numbers, which and returns a dict of page numbers, which
separate the file into new files. separate the file into new files, together
with the information whether to keep the page.
""" """
# filter all barcodes for the separator string # filter all barcodes for the separator string
# get the page numbers of the separating barcodes # get the page numbers of the separating barcodes
separator_pages = {bc.page: False for bc in barcodes if bc.is_separator}
if not settings.CONSUMER_ENABLE_ASN_BARCODE:
return separator_pages
return list({bc.page for bc in barcodes if bc.is_separator}) # add the page numbers of the ASN barcodes
# (except for first page, that might lead to infinite loops).
return {
**separator_pages,
**{bc.page: True for bc in barcodes if bc.is_asn and bc.page != 0},
}
def get_asn_from_barcodes(barcodes: List[Barcode]) -> Optional[int]: def get_asn_from_barcodes(barcodes: List[Barcode]) -> Optional[int]:
@ -242,10 +252,11 @@ def get_asn_from_barcodes(barcodes: List[Barcode]) -> Optional[int]:
return asn return asn
def separate_pages(filepath: str, pages_to_split_on: List[int]) -> List[str]: def separate_pages(filepath: str, pages_to_split_on: Dict[int, bool]) -> List[str]:
""" """
Separate the provided pdf file on the pages_to_split_on. Separate the provided pdf file on the pages_to_split_on.
The pages which are defined by page_numbers will be removed. The pages which are defined by the keys in page_numbers
will be removed if the corresponding value is false.
Returns a list of (temporary) filepaths to consume. Returns a list of (temporary) filepaths to consume.
These will need to be deleted later. These will need to be deleted later.
""" """
@ -261,26 +272,28 @@ def separate_pages(filepath: str, pages_to_split_on: List[int]) -> List[str]:
fname = os.path.splitext(os.path.basename(filepath))[0] fname = os.path.splitext(os.path.basename(filepath))[0]
pdf = Pdf.open(filepath) pdf = Pdf.open(filepath)
# Start with an empty document
current_document: List[Page] = []
# A list of documents, ie a list of lists of pages # A list of documents, ie a list of lists of pages
documents: List[List[Page]] = [] documents: List[List[Page]] = [current_document]
# A single document, ie a list of pages
document: List[Page] = []
for idx, page in enumerate(pdf.pages): for idx, page in enumerate(pdf.pages):
# Keep building the new PDF as long as it is not a # Keep building the new PDF as long as it is not a
# separator index # separator index
if idx not in pages_to_split_on: if idx not in pages_to_split_on:
document.append(page) current_document.append(page)
# Make sure to append the very last document to the documents continue
if idx == (len(pdf.pages) - 1):
documents.append(document) # This is a split index
document = [] # Start a new destination page listing
else: logger.debug(f"Starting new document at idx {idx}")
# This is a split index, save the current PDF pages, and restart current_document = []
# a new destination page listing documents.append(current_document)
logger.debug(f"Starting new document at idx {idx}") keep_page = pages_to_split_on[idx]
documents.append(document) if keep_page:
document = [] # Keep the page
# (new document is started by asn barcode)
current_document.append(page)
documents = [x for x in documents if len(x)] documents = [x for x in documents if len(x)]
@ -312,11 +325,10 @@ def save_to_dir(
Optionally rename the file. Optionally rename the file.
""" """
if os.path.isfile(filepath) and os.path.isdir(target_dir): if os.path.isfile(filepath) and os.path.isdir(target_dir):
dst = shutil.copy(filepath, target_dir) dest = target_dir
logging.debug(f"saved {str(filepath)} to {str(dst)}") if newname is not None:
if newname: dest = os.path.join(dest, newname)
dst_new = os.path.join(target_dir, newname) shutil.copy(filepath, dest)
logger.debug(f"moving {str(dst)} to {str(dst_new)}") logging.debug(f"saved {str(filepath)} to {str(dest)}")
os.rename(dst, dst_new)
else: else:
logger.warning(f"{str(filepath)} or {str(target_dir)} don't exist.") logger.warning(f"{str(filepath)} or {str(target_dir)} don't exist.")

View File

@ -146,11 +146,16 @@ class Consumer(LoggingMixin):
return return
# Validate the range is above zero and less than uint32_t max # Validate the range is above zero and less than uint32_t max
# otherwise, Whoosh can't handle it in the index # otherwise, Whoosh can't handle it in the index
if self.override_asn < 0 or self.override_asn > 0xFF_FF_FF_FF: if (
self.override_asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
or self.override_asn > Document.ARCHIVE_SERIAL_NUMBER_MAX
):
self._fail( self._fail(
MESSAGE_ASN_RANGE, MESSAGE_ASN_RANGE,
f"Not consuming {self.filename}: " f"Not consuming {self.filename}: "
f"Given ASN {self.override_asn} is out of range [0, 4,294,967,295]", f"Given ASN {self.override_asn} is out of range "
f"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, "
f"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}]",
) )
if Document.objects.filter(archive_serial_number=self.override_asn).exists(): if Document.objects.filter(archive_serial_number=self.override_asn).exists():
self._fail( self._fail(
@ -337,6 +342,7 @@ class Consumer(LoggingMixin):
mime_type, mime_type,
) )
if not parser_class: if not parser_class:
tempdir.cleanup()
self._fail(MESSAGE_UNSUPPORTED_TYPE, f"Unsupported mime type {mime_type}") self._fail(MESSAGE_UNSUPPORTED_TYPE, f"Unsupported mime type {mime_type}")
# Notify all listeners that we're going to do some work. # Notify all listeners that we're going to do some work.
@ -395,6 +401,7 @@ class Consumer(LoggingMixin):
except ParseError as e: except ParseError as e:
document_parser.cleanup() document_parser.cleanup()
tempdir.cleanup()
self._fail( self._fail(
str(e), str(e),
f"Error while consuming document {self.filename}: {e}", f"Error while consuming document {self.filename}: {e}",

View File

@ -5,6 +5,7 @@ from contextlib import contextmanager
from dateutil.parser import isoparse from dateutil.parser import isoparse
from django.conf import settings from django.conf import settings
from django.utils import timezone
from documents.models import Comment from documents.models import Comment
from documents.models import Document from documents.models import Document
from whoosh import classify from whoosh import classify
@ -89,10 +90,22 @@ def open_index_searcher():
searcher.close() searcher.close()
def update_document(writer, doc): def update_document(writer: AsyncWriter, doc: Document):
tags = ",".join([t.name for t in doc.tags.all()]) tags = ",".join([t.name for t in doc.tags.all()])
tags_ids = ",".join([str(t.id) for t in doc.tags.all()]) tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
comments = ",".join([str(c.comment) for c in Comment.objects.filter(document=doc)]) comments = ",".join([str(c.comment) for c in Comment.objects.filter(document=doc)])
asn = doc.archive_serial_number
if asn is not None and (
asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
or asn > Document.ARCHIVE_SERIAL_NUMBER_MAX
):
logger.error(
f"Not indexing Archive Serial Number {asn} of document {doc.pk}. "
f"ASN is out of range "
f"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, "
f"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}.",
)
asn = 0
writer.update_document( writer.update_document(
id=doc.pk, id=doc.pk,
title=doc.title, title=doc.title,
@ -108,7 +121,7 @@ def update_document(writer, doc):
has_type=doc.document_type is not None, has_type=doc.document_type is not None,
created=doc.created, created=doc.created,
added=doc.added, added=doc.added,
asn=doc.archive_serial_number, asn=asn,
modified=doc.modified, modified=doc.modified,
path=doc.storage_path.name if doc.storage_path else None, path=doc.storage_path.name if doc.storage_path else None,
path_id=doc.storage_path.id if doc.storage_path else None, path_id=doc.storage_path.id if doc.storage_path else None,
@ -262,7 +275,7 @@ class DelayedFullTextQuery(DelayedQuery):
["content", "title", "correspondent", "tag", "type", "comments"], ["content", "title", "correspondent", "tag", "type", "comments"],
self.searcher.ixreader.schema, self.searcher.ixreader.schema,
) )
qp.add_plugin(DateParserPlugin()) qp.add_plugin(DateParserPlugin(basedate=timezone.now()))
q = qp.parse(q_str) q = qp.parse(q_str)
corrected = self.searcher.correct_query(q, q_str) corrected = self.searcher.correct_query(q, q_str)

View File

@ -1,5 +1,6 @@
import logging import logging
import os import os
from fnmatch import filter
from pathlib import Path from pathlib import Path
from pathlib import PurePath from pathlib import PurePath
from threading import Event from threading import Event
@ -7,6 +8,7 @@ from threading import Thread
from time import monotonic from time import monotonic
from time import sleep from time import sleep
from typing import Final from typing import Final
from typing import Set
from django.conf import settings from django.conf import settings
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
@ -25,15 +27,15 @@ except ImportError: # pragma: nocover
logger = logging.getLogger("paperless.management.consumer") logger = logging.getLogger("paperless.management.consumer")
def _tags_from_path(filepath): def _tags_from_path(filepath) -> Set[Tag]:
"""Walk up the directory tree from filepath to CONSUMPTION_DIR """
and get or create Tag IDs for every directory. Walk up the directory tree from filepath to CONSUMPTION_DIR
and get or create Tag IDs for every directory.
Returns set of Tag models
""" """
normalized_consumption_dir = os.path.abspath(
os.path.normpath(settings.CONSUMPTION_DIR),
)
tag_ids = set() tag_ids = set()
path_parts = Path(filepath).relative_to(normalized_consumption_dir).parent.parts path_parts = Path(filepath).relative_to(settings.CONSUMPTION_DIR).parent.parts
for part in path_parts: for part in path_parts:
tag_ids.add( tag_ids.add(
Tag.objects.get_or_create(name__iexact=part, defaults={"name": part})[0].pk, Tag.objects.get_or_create(name__iexact=part, defaults={"name": part})[0].pk,
@ -43,14 +45,41 @@ def _tags_from_path(filepath):
def _is_ignored(filepath: str) -> bool: def _is_ignored(filepath: str) -> bool:
normalized_consumption_dir = os.path.abspath( """
os.path.normpath(settings.CONSUMPTION_DIR), Checks if the given file should be ignored, based on configured
patterns.
Returns True if the file is ignored, False otherwise
"""
filepath = os.path.abspath(
os.path.normpath(filepath),
) )
filepath_relative = PurePath(filepath).relative_to(normalized_consumption_dir)
return any(filepath_relative.match(p) for p in settings.CONSUMER_IGNORE_PATTERNS) # Trim out the consume directory, leaving only filename and it's
# path relative to the consume directory
filepath_relative = PurePath(filepath).relative_to(settings.CONSUMPTION_DIR)
# March through the components of the path, including directories and the filename
# looking for anything matching
# foo/bar/baz/file.pdf -> (foo, bar, baz, file.pdf)
parts = []
for part in filepath_relative.parts:
# If the part is not the name (ie, it's a dir)
# Need to append the trailing slash or fnmatch doesn't match
# fnmatch("dir", "dir/*") == False
# fnmatch("dir/", "dir/*") == True
if part != filepath_relative.name:
part = part + "/"
parts.append(part)
for pattern in settings.CONSUMER_IGNORE_PATTERNS:
if len(filter(parts, pattern)):
return True
return False
def _consume(filepath): def _consume(filepath: str) -> None:
if os.path.isdir(filepath) or _is_ignored(filepath): if os.path.isdir(filepath) or _is_ignored(filepath):
return return
@ -103,7 +132,13 @@ def _consume(filepath):
logger.exception("Error while consuming document") logger.exception("Error while consuming document")
def _consume_wait_unmodified(file): def _consume_wait_unmodified(file: str) -> None:
"""
Waits for the given file to appear unmodified based on file size
and modification time. Will wait a configured number of seconds
and retry a configured number of times before either consuming or
giving up
"""
if _is_ignored(file): if _is_ignored(file):
return return

View File

@ -311,8 +311,8 @@ class Command(BaseCommand):
archive_target = None archive_target = None
# 3.4. write files to target folder # 3.4. write files to target folder
t = int(time.mktime(document.created.timetuple()))
if document.storage_type == Document.STORAGE_TYPE_GPG: if document.storage_type == Document.STORAGE_TYPE_GPG:
t = int(time.mktime(document.created.timetuple()))
original_target.parent.mkdir(parents=True, exist_ok=True) original_target.parent.mkdir(parents=True, exist_ok=True)
with document.source_file as out_file: with document.source_file as out_file:

View File

@ -0,0 +1,23 @@
# Generated by Django 4.1.5 on 2023-02-03 21:53
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("documents", "1029_alter_document_archive_serial_number"),
]
operations = [
migrations.AlterField(
model_name="paperlesstask",
name="task_file_name",
field=models.CharField(
help_text="Name of the file which the Task was run for",
max_length=255,
null=True,
verbose_name="Task Filename",
),
),
]

View File

@ -3,6 +3,7 @@ import logging
import os import os
import re import re
from collections import OrderedDict from collections import OrderedDict
from typing import Final
from typing import Optional from typing import Optional
import dateutil.parser import dateutil.parser
@ -229,6 +230,9 @@ class Document(models.Model):
help_text=_("The original name of the file when it was uploaded"), help_text=_("The original name of the file when it was uploaded"),
) )
ARCHIVE_SERIAL_NUMBER_MIN: Final[int] = 0
ARCHIVE_SERIAL_NUMBER_MAX: Final[int] = 0xFF_FF_FF_FF
archive_serial_number = models.PositiveIntegerField( archive_serial_number = models.PositiveIntegerField(
_("archive serial number"), _("archive serial number"),
blank=True, blank=True,
@ -236,8 +240,8 @@ class Document(models.Model):
unique=True, unique=True,
db_index=True, db_index=True,
validators=[ validators=[
MaxValueValidator(0xFF_FF_FF_FF), MaxValueValidator(ARCHIVE_SERIAL_NUMBER_MAX),
MinValueValidator(0), MinValueValidator(ARCHIVE_SERIAL_NUMBER_MIN),
], ],
help_text=_( help_text=_(
"The position of this document in your physical document " "archive.", "The position of this document in your physical document " "archive.",
@ -555,7 +559,7 @@ class PaperlessTask(models.Model):
task_file_name = models.CharField( task_file_name = models.CharField(
null=True, null=True,
max_length=255, max_length=255,
verbose_name=_("Task Name"), verbose_name=_("Task Filename"),
help_text=_("Name of the file which the Task was run for"), help_text=_("Name of the file which the Task was run for"),
) )

View File

@ -599,11 +599,17 @@ class StoragePathSerializer(MatchingModelSerializer):
document_type="document_type", document_type="document_type",
created="created", created="created",
created_year="created_year", created_year="created_year",
created_year_short="created_year_short",
created_month="created_month", created_month="created_month",
created_month_name="created_month_name",
created_month_name_short="created_month_name_short",
created_day="created_day", created_day="created_day",
added="added", added="added",
added_year="added_year", added_year="added_year",
added_year_short="added_year_short",
added_month="added_month", added_month="added_month",
added_month_name="added_month_name",
added_month_name_short="added_month_name_short",
added_day="added_day", added_day="added_day",
asn="asn", asn="asn",
tags="tags", tags="tags",

View File

@ -128,6 +128,18 @@ def consume_file(
) )
if document_list: if document_list:
# If the file is an upload, it's in the scratch directory
# Move it to consume directory to be picked up
# Otherwise, use the current parent to keep possible tags
# from subdirectories
try:
# is_relative_to would be nicer, but new in 3.9
_ = path.relative_to(settings.SCRATCH_DIR)
save_to_dir = settings.CONSUMPTION_DIR
except ValueError:
save_to_dir = path.parent
for n, document in enumerate(document_list): for n, document in enumerate(document_list):
# save to consumption dir # save to consumption dir
# rename it to the original filename with number prefix # rename it to the original filename with number prefix
@ -136,23 +148,18 @@ def consume_file(
else: else:
newname = None newname = None
# If the file is an upload, it's in the scratch directory
# Move it to consume directory to be picked up
# Otherwise, use the current parent to keep possible tags
# from subdirectories
try:
# is_relative_to would be nicer, but new in 3.9
_ = path.relative_to(settings.SCRATCH_DIR)
save_to_dir = settings.CONSUMPTION_DIR
except ValueError:
save_to_dir = path.parent
barcodes.save_to_dir( barcodes.save_to_dir(
document, document,
newname=newname, newname=newname,
target_dir=save_to_dir, target_dir=save_to_dir,
) )
# Split file has been copied safely, remove it
os.remove(document)
# And clean up the directory as well, now it's empty
shutil.rmtree(os.path.dirname(document_list[0]))
# Delete the PDF file which was split # Delete the PDF file which was split
os.remove(doc_barcode_info.pdf_path) os.remove(doc_barcode_info.pdf_path)
@ -164,7 +171,7 @@ def consume_file(
# notify the sender, otherwise the progress bar # notify the sender, otherwise the progress bar
# in the UI stays stuck # in the UI stays stuck
payload = { payload = {
"filename": override_filename, "filename": override_filename or path.name,
"task_id": task_id, "task_id": task_id,
"current_progress": 100, "current_progress": 100,
"max_progress": 100, "max_progress": 100,

View File

@ -7,6 +7,7 @@ import tempfile
import urllib.request import urllib.request
import uuid import uuid
import zipfile import zipfile
from datetime import timedelta
from pathlib import Path from pathlib import Path
from unittest import mock from unittest import mock
from unittest.mock import MagicMock from unittest.mock import MagicMock
@ -23,6 +24,7 @@ from django.conf import settings
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.test import override_settings from django.test import override_settings
from django.utils import timezone from django.utils import timezone
from dateutil.relativedelta import relativedelta
from documents import bulk_edit from documents import bulk_edit
from documents import index from documents import index
from documents.models import Correspondent from documents.models import Correspondent
@ -119,28 +121,28 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
response = self.client.get("/api/documents/", format="json") response = self.client.get("/api/documents/", format="json")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
results_full = response.data["results"] results_full = response.data["results"]
self.assertTrue("content" in results_full[0]) self.assertIn("content", results_full[0])
self.assertTrue("id" in results_full[0]) self.assertIn("id", results_full[0])
response = self.client.get("/api/documents/?fields=id", format="json") response = self.client.get("/api/documents/?fields=id", format="json")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
results = response.data["results"] results = response.data["results"]
self.assertFalse("content" in results[0]) self.assertFalse("content" in results[0])
self.assertTrue("id" in results[0]) self.assertIn("id", results[0])
self.assertEqual(len(results[0]), 1) self.assertEqual(len(results[0]), 1)
response = self.client.get("/api/documents/?fields=content", format="json") response = self.client.get("/api/documents/?fields=content", format="json")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
results = response.data["results"] results = response.data["results"]
self.assertTrue("content" in results[0]) self.assertIn("content", results[0])
self.assertFalse("id" in results[0]) self.assertFalse("id" in results[0])
self.assertEqual(len(results[0]), 1) self.assertEqual(len(results[0]), 1)
response = self.client.get("/api/documents/?fields=id,content", format="json") response = self.client.get("/api/documents/?fields=id,content", format="json")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
results = response.data["results"] results = response.data["results"]
self.assertTrue("content" in results[0]) self.assertIn("content", results[0])
self.assertTrue("id" in results[0]) self.assertIn("id", results[0])
self.assertEqual(len(results[0]), 2) self.assertEqual(len(results[0]), 2)
response = self.client.get( response = self.client.get(
@ -150,7 +152,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
results = response.data["results"] results = response.data["results"]
self.assertFalse("content" in results[0]) self.assertFalse("content" in results[0])
self.assertTrue("id" in results[0]) self.assertIn("id", results[0])
self.assertEqual(len(results[0]), 1) self.assertEqual(len(results[0]), 1)
response = self.client.get("/api/documents/?fields=", format="json") response = self.client.get("/api/documents/?fields=", format="json")
@ -505,6 +507,270 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
response = self.client.get("/api/documents/?query=content&page=3&page_size=10") response = self.client.get("/api/documents/?query=content&page=3&page_size=10")
self.assertEqual(response.status_code, 404) self.assertEqual(response.status_code, 404)
@override_settings(
TIME_ZONE="UTC",
)
def test_search_added_in_last_week(self):
"""
GIVEN:
- Three documents added right now
- The timezone is UTC time
WHEN:
- Query for documents added in the last 7 days
THEN:
- All three recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 week to now]")
results = response.data["results"]
# Expect 3 documents returned
self.assertEqual(len(results), 3)
for idx, subset in enumerate(
[
{"id": 1, "title": "invoice"},
{"id": 2, "title": "bank statement 1"},
{"id": 3, "title": "bank statement 3"},
],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
@override_settings(
TIME_ZONE="America/Chicago",
)
def test_search_added_in_last_week_with_timezone_behind(self):
"""
GIVEN:
- Two documents added right now
- One document added over a week ago
- The timezone is behind UTC time (-6)
WHEN:
- Query for documents added in the last 7 days
THEN:
- The two recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
# 7 days, 1 hour and 1 minute ago
added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 week to now]")
results = response.data["results"]
# Expect 2 documents returned
self.assertEqual(len(results), 2)
for idx, subset in enumerate(
[{"id": 1, "title": "invoice"}, {"id": 2, "title": "bank statement 1"}],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
@override_settings(
TIME_ZONE="Europe/Sofia",
)
def test_search_added_in_last_week_with_timezone_ahead(self):
"""
GIVEN:
- Two documents added right now
- One document added over a week ago
- The timezone is behind UTC time (+2)
WHEN:
- Query for documents added in the last 7 days
THEN:
- The two recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
# 7 days, 1 hour and 1 minute ago
added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 week to now]")
results = response.data["results"]
# Expect 2 documents returned
self.assertEqual(len(results), 2)
for idx, subset in enumerate(
[{"id": 1, "title": "invoice"}, {"id": 2, "title": "bank statement 1"}],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
def test_search_added_in_last_month(self):
"""
GIVEN:
- One document added right now
- One documents added about a week ago
- One document added over 1 month
WHEN:
- Query for documents added in the last month
THEN:
- The two recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
# 1 month, 1 day ago
added=timezone.now() - relativedelta(months=1, days=1),
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
# 7 days, 1 hour and 1 minute ago
added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 month to now]")
results = response.data["results"]
# Expect 2 documents returned
self.assertEqual(len(results), 2)
for idx, subset in enumerate(
[{"id": 1, "title": "invoice"}, {"id": 3, "title": "bank statement 3"}],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
@override_settings(
TIME_ZONE="America/Denver",
)
def test_search_added_in_last_month_timezone_behind(self):
"""
GIVEN:
- One document added right now
- One documents added about a week ago
- One document added over 1 month
- The timezone is behind UTC time (-6 or -7)
WHEN:
- Query for documents added in the last month
THEN:
- The two recent documents are returned
"""
d1 = Document.objects.create(
title="invoice",
content="the thing i bought at a shop and paid with bank account",
checksum="A",
pk=1,
)
d2 = Document.objects.create(
title="bank statement 1",
content="things i paid for in august",
pk=2,
checksum="B",
# 1 month, 1 day ago
added=timezone.now() - relativedelta(months=1, days=1),
)
d3 = Document.objects.create(
title="bank statement 3",
content="things i paid for in september",
pk=3,
checksum="C",
# 7 days, 1 hour and 1 minute ago
added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
)
with index.open_index_writer() as writer:
index.update_document(writer, d1)
index.update_document(writer, d2)
index.update_document(writer, d3)
response = self.client.get("/api/documents/?query=added:[-1 month to now]")
results = response.data["results"]
# Expect 2 documents returned
self.assertEqual(len(results), 2)
for idx, subset in enumerate(
[{"id": 1, "title": "invoice"}, {"id": 3, "title": "bank statement 3"}],
):
result = results[idx]
# Assert subset in results
self.assertDictEqual(result, {**result, **subset})
@mock.patch("documents.index.autocomplete") @mock.patch("documents.index.autocomplete")
def test_search_autocomplete(self, m): def test_search_autocomplete(self, m):
m.side_effect = lambda ix, term, limit: [term for _ in range(limit)] m.side_effect = lambda ix, term, limit: [term for _ in range(limit)]
@ -2933,8 +3199,32 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, 400) self.assertEqual(response.status_code, 400)
self.assertEqual(StoragePath.objects.count(), 1) self.assertEqual(StoragePath.objects.count(), 1)
def test_api_storage_path_placeholders(self):
"""
GIVEN:
- API request to create a storage path with placeholders
- Storage path is valid
WHEN:
- API is called
THEN:
- Correct HTTP response
- New storage path is created
"""
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"name": "Storage path with placeholders",
"path": "{title}/{correspondent}/{document_type}/{created}/{created_year}/{created_year_short}/{created_month}/{created_month_name}/{created_month_name_short}/{created_day}/{added}/{added_year}/{added_year_short}/{added_month}/{added_month_name}/{added_month_name_short}/{added_day}/{asn}/{tags}/{tag_list}/",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, 201)
self.assertEqual(StoragePath.objects.count(), 2)
class TestTasks(APITestCase):
class TestTasks(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/tasks/" ENDPOINT = "/api/tasks/"
ENDPOINT_ACKNOWLEDGE = "/api/acknowledge_tasks/" ENDPOINT_ACKNOWLEDGE = "/api/acknowledge_tasks/"

View File

@ -294,7 +294,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0]) self.assertDictEqual(separator_page_numbers, {0: False})
def test_scan_file_for_separating_barcodes_none_present(self): def test_scan_file_for_separating_barcodes_none_present(self):
""" """
@ -314,7 +314,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, []) self.assertDictEqual(separator_page_numbers, {})
def test_scan_file_for_separating_barcodes_middle_page(self): def test_scan_file_for_separating_barcodes_middle_page(self):
""" """
@ -337,7 +337,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [1]) self.assertDictEqual(separator_page_numbers, {1: False})
def test_scan_file_for_separating_barcodes_multiple_pages(self): def test_scan_file_for_separating_barcodes_multiple_pages(self):
""" """
@ -360,7 +360,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [2, 5]) self.assertDictEqual(separator_page_numbers, {2: False, 5: False})
def test_scan_file_for_separating_barcodes_upside_down(self): def test_scan_file_for_separating_barcodes_upside_down(self):
""" """
@ -384,7 +384,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [1]) self.assertDictEqual(separator_page_numbers, {1: False})
def test_scan_file_for_separating_barcodes_fax_decode(self): def test_scan_file_for_separating_barcodes_fax_decode(self):
""" """
@ -407,7 +407,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [1]) self.assertDictEqual(separator_page_numbers, {1: False})
def test_scan_file_for_separating_qr_barcodes(self): def test_scan_file_for_separating_qr_barcodes(self):
""" """
@ -431,7 +431,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0]) self.assertDictEqual(separator_page_numbers, {0: False})
@override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE") @override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE")
def test_scan_file_for_separating_custom_barcodes(self): def test_scan_file_for_separating_custom_barcodes(self):
@ -456,7 +456,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0]) self.assertDictEqual(separator_page_numbers, {0: False})
@override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE") @override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE")
def test_scan_file_for_separating_custom_qr_barcodes(self): def test_scan_file_for_separating_custom_qr_barcodes(self):
@ -482,7 +482,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0]) self.assertDictEqual(separator_page_numbers, {0: False})
@override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE") @override_settings(CONSUMER_BARCODE_STRING="CUSTOM BARCODE")
def test_scan_file_for_separating_custom_128_barcodes(self): def test_scan_file_for_separating_custom_128_barcodes(self):
@ -508,7 +508,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, [0]) self.assertDictEqual(separator_page_numbers, {0: False})
def test_scan_file_for_separating_wrong_qr_barcodes(self): def test_scan_file_for_separating_wrong_qr_barcodes(self):
""" """
@ -533,7 +533,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, []) self.assertDictEqual(separator_page_numbers, {})
@override_settings(CONSUMER_BARCODE_STRING="ADAR-NEXTDOC") @override_settings(CONSUMER_BARCODE_STRING="ADAR-NEXTDOC")
def test_scan_file_for_separating_qr_barcodes(self): def test_scan_file_for_separating_qr_barcodes(self):
@ -558,7 +558,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertGreater(len(doc_barcode_info.barcodes), 0) self.assertGreater(len(doc_barcode_info.barcodes), 0)
self.assertListEqual(separator_page_numbers, [1]) self.assertDictEqual(separator_page_numbers, {1: False})
def test_separate_pages(self): def test_separate_pages(self):
""" """
@ -573,7 +573,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
self.BARCODE_SAMPLE_DIR, self.BARCODE_SAMPLE_DIR,
"patch-code-t-middle.pdf", "patch-code-t-middle.pdf",
) )
documents = barcodes.separate_pages(test_file, [1]) documents = barcodes.separate_pages(test_file, {1: False})
self.assertEqual(len(documents), 2) self.assertEqual(len(documents), 2)
@ -591,7 +591,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
self.BARCODE_SAMPLE_DIR, self.BARCODE_SAMPLE_DIR,
"patch-code-t-double.pdf", "patch-code-t-double.pdf",
) )
pages = barcodes.separate_pages(test_file, [1, 2]) pages = barcodes.separate_pages(test_file, {1: False, 2: False})
self.assertEqual(len(pages), 2) self.assertEqual(len(pages), 2)
@ -610,7 +610,7 @@ class TestBarcode(DirectoriesMixin, TestCase):
"patch-code-t-middle.pdf", "patch-code-t-middle.pdf",
) )
with self.assertLogs("paperless.barcodes", level="WARNING") as cm: with self.assertLogs("paperless.barcodes", level="WARNING") as cm:
pages = barcodes.separate_pages(test_file, []) pages = barcodes.separate_pages(test_file, {})
self.assertEqual(pages, []) self.assertEqual(pages, [])
self.assertEqual( self.assertEqual(
cm.output, cm.output,
@ -858,7 +858,88 @@ class TestBarcode(DirectoriesMixin, TestCase):
) )
self.assertEqual(doc_barcode_info.pdf_path, test_file) self.assertEqual(doc_barcode_info.pdf_path, test_file)
self.assertListEqual(separator_page_numbers, []) self.assertDictEqual(separator_page_numbers, {})
@override_settings(
CONSUMER_ENABLE_BARCODES=True,
CONSUMER_ENABLE_ASN_BARCODE=True,
)
def test_separate_pages_by_asn_barcodes_and_patcht(self):
"""
GIVEN:
- Input PDF with a patch code on page 3 and ASN barcodes on pages 1,5,6,9,11
WHEN:
- Input file is split on barcodes
THEN:
- Correct number of files produced, split correctly by correct pages
"""
test_file = os.path.join(
os.path.dirname(__file__),
self.BARCODE_SAMPLE_DIR,
"split-by-asn-2.pdf",
)
doc_barcode_info = barcodes.scan_file_for_barcodes(
test_file,
)
separator_page_numbers = barcodes.get_separating_barcodes(
doc_barcode_info.barcodes,
)
self.assertEqual(test_file, doc_barcode_info.pdf_path)
self.assertDictEqual(
separator_page_numbers,
{
2: False,
4: True,
5: True,
8: True,
10: True,
},
)
document_list = barcodes.separate_pages(test_file, separator_page_numbers)
self.assertEqual(len(document_list), 6)
@override_settings(
CONSUMER_ENABLE_BARCODES=True,
CONSUMER_ENABLE_ASN_BARCODE=True,
)
def test_separate_pages_by_asn_barcodes(self):
"""
GIVEN:
- Input PDF with ASN barcodes on pages 1,3,4,7,9
WHEN:
- Input file is split on barcodes
THEN:
- Correct number of files produced, split correctly by correct pages
"""
test_file = os.path.join(
os.path.dirname(__file__),
self.BARCODE_SAMPLE_DIR,
"split-by-asn-1.pdf",
)
doc_barcode_info = barcodes.scan_file_for_barcodes(
test_file,
)
separator_page_numbers = barcodes.get_separating_barcodes(
doc_barcode_info.barcodes,
)
self.assertEqual(test_file, doc_barcode_info.pdf_path)
self.assertDictEqual(
separator_page_numbers,
{
2: True,
3: True,
6: True,
8: True,
},
)
document_list = barcodes.separate_pages(test_file, separator_page_numbers)
self.assertEqual(len(document_list), 5)
class TestAsnBarcodes(DirectoriesMixin, TestCase): class TestAsnBarcodes(DirectoriesMixin, TestCase):

View File

@ -847,13 +847,11 @@ class PreConsumeTestCase(TestCase):
self.assertEqual(command[0], script.name) self.assertEqual(command[0], script.name)
self.assertEqual(command[1], "path-to-file") self.assertEqual(command[1], "path-to-file")
self.assertDictContainsSubset( subset = {
{ "DOCUMENT_SOURCE_PATH": c.original_path,
"DOCUMENT_SOURCE_PATH": c.original_path, "DOCUMENT_WORKING_PATH": c.path,
"DOCUMENT_WORKING_PATH": c.path, }
}, self.assertDictEqual(environment, {**environment, **subset})
environment,
)
@mock.patch("documents.consumer.Consumer.log") @mock.patch("documents.consumer.Consumer.log")
def test_script_with_output(self, mocked_log): def test_script_with_output(self, mocked_log):
@ -983,16 +981,15 @@ class PostConsumeTestCase(TestCase):
self.assertEqual(command[7], "my_bank") self.assertEqual(command[7], "my_bank")
self.assertCountEqual(command[8].split(","), ["a", "b"]) self.assertCountEqual(command[8].split(","), ["a", "b"])
self.assertDictContainsSubset( subset = {
{ "DOCUMENT_ID": str(doc.pk),
"DOCUMENT_ID": str(doc.pk), "DOCUMENT_DOWNLOAD_URL": f"/api/documents/{doc.pk}/download/",
"DOCUMENT_DOWNLOAD_URL": f"/api/documents/{doc.pk}/download/", "DOCUMENT_THUMBNAIL_URL": f"/api/documents/{doc.pk}/thumb/",
"DOCUMENT_THUMBNAIL_URL": f"/api/documents/{doc.pk}/thumb/", "DOCUMENT_CORRESPONDENT": "my_bank",
"DOCUMENT_CORRESPONDENT": "my_bank", "DOCUMENT_TAGS": "a,b",
"DOCUMENT_TAGS": "a,b", }
},
environment, self.assertDictEqual(environment, {**environment, **subset})
)
def test_script_exit_non_zero(self): def test_script_exit_non_zero(self):
""" """

View File

@ -25,7 +25,7 @@ class TestImporter(TestCase):
cmd.manifest = [{"model": "documents.document"}] cmd.manifest = [{"model": "documents.document"}]
with self.assertRaises(CommandError) as cm: with self.assertRaises(CommandError) as cm:
cmd._check_manifest() cmd._check_manifest()
self.assertTrue("The manifest file contains a record" in str(cm.exception)) self.assertIn("The manifest file contains a record", str(cm.exception))
cmd.manifest = [ cmd.manifest = [
{"model": "documents.document", EXPORTER_FILE_NAME: "noexist.pdf"}, {"model": "documents.document", EXPORTER_FILE_NAME: "noexist.pdf"},
@ -33,6 +33,7 @@ class TestImporter(TestCase):
# self.assertRaises(CommandError, cmd._check_manifest) # self.assertRaises(CommandError, cmd._check_manifest)
with self.assertRaises(CommandError) as cm: with self.assertRaises(CommandError) as cm:
cmd._check_manifest() cmd._check_manifest()
self.assertTrue( self.assertIn(
'The manifest file refers to "noexist.pdf"' in str(cm.exception), 'The manifest file refers to "noexist.pdf"',
str(cm.exception),
) )

View File

@ -1,3 +1,5 @@
from unittest import mock
from django.test import TestCase from django.test import TestCase
from documents import index from documents import index
from documents.models import Document from documents.models import Document
@ -31,3 +33,60 @@ class TestAutoComplete(DirectoriesMixin, TestCase):
) )
self.assertListEqual(index.autocomplete(ix, "tes", limit=1), [b"test3"]) self.assertListEqual(index.autocomplete(ix, "tes", limit=1), [b"test3"])
self.assertListEqual(index.autocomplete(ix, "tes", limit=0), []) self.assertListEqual(index.autocomplete(ix, "tes", limit=0), [])
def test_archive_serial_number_ranging(self):
"""
GIVEN:
- Document with an archive serial number above schema allowed size
WHEN:
- Document is provided to the index
THEN:
- Error is logged
- Document ASN is reset to 0 for the index
"""
doc1 = Document.objects.create(
title="doc1",
checksum="A",
content="test test2 test3",
# yes, this is allowed, unless full_clean is run
# DRF does call the validators, this test won't
archive_serial_number=Document.ARCHIVE_SERIAL_NUMBER_MAX + 1,
)
with self.assertLogs("paperless.index", level="ERROR") as cm:
with mock.patch(
"documents.index.AsyncWriter.update_document",
) as mocked_update_doc:
index.add_or_update_document(doc1)
mocked_update_doc.assert_called_once()
_, kwargs = mocked_update_doc.call_args
self.assertEqual(kwargs["asn"], 0)
error_str = cm.output[0]
expected_str = "ERROR:paperless.index:Not indexing Archive Serial Number 4294967296 of document 1"
self.assertIn(expected_str, error_str)
def test_archive_serial_number_is_none(self):
"""
GIVEN:
- Document with no archive serial number
WHEN:
- Document is provided to the index
THEN:
- ASN isn't touched
"""
doc1 = Document.objects.create(
title="doc1",
checksum="A",
content="test test2 test3",
)
with mock.patch(
"documents.index.AsyncWriter.update_document",
) as mocked_update_doc:
index.add_or_update_document(doc1)
mocked_update_doc.assert_called_once()
_, kwargs = mocked_update_doc.call_args
self.assertIsNone(kwargs["asn"])

View File

@ -247,22 +247,85 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
def test_is_ignored(self): def test_is_ignored(self):
test_paths = [ test_paths = [
(os.path.join(self.dirs.consumption_dir, "foo.pdf"), False), {
(os.path.join(self.dirs.consumption_dir, "foo", "bar.pdf"), False), "path": os.path.join(self.dirs.consumption_dir, "foo.pdf"),
(os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"), True), "ignore": False,
( },
os.path.join(self.dirs.consumption_dir, "foo", ".DS_STORE", "bar.pdf"), {
True, "path": os.path.join(self.dirs.consumption_dir, "foo", "bar.pdf"),
), "ignore": False,
(os.path.join(self.dirs.consumption_dir, ".stfolder", "foo.pdf"), True), },
(os.path.join(self.dirs.consumption_dir, "._foo.pdf"), True), {
(os.path.join(self.dirs.consumption_dir, "._foo", "bar.pdf"), False), "path": os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"),
"ignore": True,
},
{
"path": os.path.join(
self.dirs.consumption_dir,
"foo",
".DS_STORE",
"bar.pdf",
),
"ignore": True,
},
{
"path": os.path.join(
self.dirs.consumption_dir,
".DS_STORE",
"foo",
"bar.pdf",
),
"ignore": True,
},
{
"path": os.path.join(self.dirs.consumption_dir, ".stfolder", "foo.pdf"),
"ignore": True,
},
{
"path": os.path.join(self.dirs.consumption_dir, ".stfolder.pdf"),
"ignore": False,
},
{
"path": os.path.join(
self.dirs.consumption_dir,
".stversions",
"foo.pdf",
),
"ignore": True,
},
{
"path": os.path.join(self.dirs.consumption_dir, ".stversions.pdf"),
"ignore": False,
},
{
"path": os.path.join(self.dirs.consumption_dir, "._foo.pdf"),
"ignore": True,
},
{
"path": os.path.join(self.dirs.consumption_dir, "my_foo.pdf"),
"ignore": False,
},
{
"path": os.path.join(self.dirs.consumption_dir, "._foo", "bar.pdf"),
"ignore": True,
},
{
"path": os.path.join(
self.dirs.consumption_dir,
"@eaDir",
"SYNO@.fileindexdb",
"_1jk.fnm",
),
"ignore": True,
},
] ]
for file_path, expected_ignored in test_paths: for test_setup in test_paths:
filepath = test_setup["path"]
expected_ignored_result = test_setup["ignore"]
self.assertEqual( self.assertEqual(
expected_ignored, expected_ignored_result,
document_consumer._is_ignored(file_path), document_consumer._is_ignored(filepath),
f'_is_ignored("{file_path}") != {expected_ignored}', f'_is_ignored("{filepath}") != {expected_ignored_result}',
) )
@mock.patch("documents.management.commands.document_consumer.open") @mock.patch("documents.management.commands.document_consumer.open")

View File

@ -1,6 +1,8 @@
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
from unittest import mock from unittest import mock
from django.apps import apps
from django.test import override_settings
from django.test import TestCase from django.test import TestCase
from documents.parsers import get_default_file_extension from documents.parsers import get_default_file_extension
from documents.parsers import get_parser_class_for_mime_type from documents.parsers import get_parser_class_for_mime_type
@ -8,6 +10,7 @@ from documents.parsers import get_supported_file_extensions
from documents.parsers import is_file_ext_supported from documents.parsers import is_file_ext_supported
from paperless_tesseract.parsers import RasterisedDocumentParser from paperless_tesseract.parsers import RasterisedDocumentParser
from paperless_text.parsers import TextDocumentParser from paperless_text.parsers import TextDocumentParser
from paperless_tika.parsers import TikaDocumentParser
class TestParserDiscovery(TestCase): class TestParserDiscovery(TestCase):
@ -124,14 +127,43 @@ class TestParserDiscovery(TestCase):
class TestParserAvailability(TestCase): class TestParserAvailability(TestCase):
def test_file_extensions(self): def test_tesseract_parser(self):
"""
GIVEN:
- Various mime types
WHEN:
- The parser class is instantiated
THEN:
- The Tesseract based parser is return
"""
supported_mimes_and_exts = [ supported_mimes_and_exts = [
("application/pdf", ".pdf"), ("application/pdf", ".pdf"),
("image/png", ".png"), ("image/png", ".png"),
("image/jpeg", ".jpg"), ("image/jpeg", ".jpg"),
("image/tiff", ".tif"), ("image/tiff", ".tif"),
("image/webp", ".webp"), ("image/webp", ".webp"),
]
supported_exts = get_supported_file_extensions()
for mime_type, ext in supported_mimes_and_exts:
self.assertIn(ext, supported_exts)
self.assertEqual(get_default_file_extension(mime_type), ext)
self.assertIsInstance(
get_parser_class_for_mime_type(mime_type)(logging_group=None),
RasterisedDocumentParser,
)
def test_text_parser(self):
"""
GIVEN:
- Various mime types of a text form
WHEN:
- The parser class is instantiated
THEN:
- The text based parser is return
"""
supported_mimes_and_exts = [
("text/plain", ".txt"), ("text/plain", ".txt"),
("text/csv", ".csv"), ("text/csv", ".csv"),
] ]
@ -141,23 +173,55 @@ class TestParserAvailability(TestCase):
for mime_type, ext in supported_mimes_and_exts: for mime_type, ext in supported_mimes_and_exts:
self.assertIn(ext, supported_exts) self.assertIn(ext, supported_exts)
self.assertEqual(get_default_file_extension(mime_type), ext) self.assertEqual(get_default_file_extension(mime_type), ext)
self.assertIsInstance(
get_parser_class_for_mime_type(mime_type)(logging_group=None),
TextDocumentParser,
)
def test_tika_parser(self):
"""
GIVEN:
- Various mime types of a office document form
WHEN:
- The parser class is instantiated
THEN:
- The Tika/Gotenberg based parser is return
"""
supported_mimes_and_exts = [
("application/vnd.oasis.opendocument.text", ".odt"),
("text/rtf", ".rtf"),
("application/msword", ".doc"),
(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".docx",
),
]
# Force the app ready to notice the settings override
with override_settings(TIKA_ENABLED=True, INSTALLED_APPS=["paperless_tika"]):
app = apps.get_app_config("paperless_tika")
app.ready()
supported_exts = get_supported_file_extensions()
for mime_type, ext in supported_mimes_and_exts:
self.assertIn(ext, supported_exts)
self.assertEqual(get_default_file_extension(mime_type), ext)
self.assertIsInstance(
get_parser_class_for_mime_type(mime_type)(logging_group=None),
TikaDocumentParser,
)
def test_no_parser_for_mime(self):
self.assertIsNone(get_parser_class_for_mime_type("text/sdgsdf"))
def test_default_extension(self):
# Test no parser declared still returns a an extension # Test no parser declared still returns a an extension
self.assertEqual(get_default_file_extension("application/zip"), ".zip") self.assertEqual(get_default_file_extension("application/zip"), ".zip")
# Test invalid mimetype returns no extension # Test invalid mimetype returns no extension
self.assertEqual(get_default_file_extension("aasdasd/dgfgf"), "") self.assertEqual(get_default_file_extension("aasdasd/dgfgf"), "")
self.assertIsInstance( def test_file_extension_support(self):
get_parser_class_for_mime_type("application/pdf")(logging_group=None),
RasterisedDocumentParser,
)
self.assertIsInstance(
get_parser_class_for_mime_type("text/plain")(logging_group=None),
TextDocumentParser,
)
self.assertIsNone(get_parser_class_for_mime_type("text/sdgsdf"))
self.assertTrue(is_file_ext_supported(".pdf")) self.assertTrue(is_file_ext_supported(".pdf"))
self.assertFalse(is_file_ext_supported(".hsdfh")) self.assertFalse(is_file_ext_supported(".hsdfh"))
self.assertFalse(is_file_ext_supported("")) self.assertFalse(is_file_ext_supported(""))

View File

@ -3,7 +3,7 @@ msgstr ""
"Project-Id-Version: paperless-ngx\n" "Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n" "Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2022-11-09 21:50+0000\n" "POT-Creation-Date: 2022-11-09 21:50+0000\n"
"PO-Revision-Date: 2023-01-23 12:37\n" "PO-Revision-Date: 2023-01-27 19:22\n"
"Last-Translator: \n" "Last-Translator: \n"
"Language-Team: Dutch\n" "Language-Team: Dutch\n"
"Language: nl_NL\n" "Language: nl_NL\n"
@ -368,15 +368,15 @@ msgstr "heeft tags in"
#: documents/models.py:410 #: documents/models.py:410
msgid "ASN greater than" msgid "ASN greater than"
msgstr "" msgstr "ASN groter dan"
#: documents/models.py:411 #: documents/models.py:411
msgid "ASN less than" msgid "ASN less than"
msgstr "" msgstr "ASN kleiner dan"
#: documents/models.py:412 #: documents/models.py:412
msgid "storage path is" msgid "storage path is"
msgstr "" msgstr "opslagpad is"
#: documents/models.py:422 #: documents/models.py:422
msgid "rule type" msgid "rule type"
@ -396,99 +396,99 @@ msgstr "filterregels"
#: documents/models.py:536 #: documents/models.py:536
msgid "Task ID" msgid "Task ID"
msgstr "" msgstr "Taak ID"
#: documents/models.py:537 #: documents/models.py:537
msgid "Celery ID for the Task that was run" msgid "Celery ID for the Task that was run"
msgstr "" msgstr "Celery ID voor de taak die werd uitgevoerd"
#: documents/models.py:542 #: documents/models.py:542
msgid "Acknowledged" msgid "Acknowledged"
msgstr "" msgstr "Bevestigd"
#: documents/models.py:543 #: documents/models.py:543
msgid "If the task is acknowledged via the frontend or API" msgid "If the task is acknowledged via the frontend or API"
msgstr "" msgstr "Of de taak is bevestigd via de frontend of de API"
#: documents/models.py:549 documents/models.py:556 #: documents/models.py:549 documents/models.py:556
msgid "Task Name" msgid "Task Name"
msgstr "" msgstr "Taaknaam"
#: documents/models.py:550 #: documents/models.py:550
msgid "Name of the file which the Task was run for" msgid "Name of the file which the Task was run for"
msgstr "" msgstr "Naam van het bestand waarvoor de taak werd uitgevoerd"
#: documents/models.py:557 #: documents/models.py:557
msgid "Name of the Task which was run" msgid "Name of the Task which was run"
msgstr "" msgstr "Naam van de uitgevoerde taak"
#: documents/models.py:562 #: documents/models.py:562
msgid "Task Positional Arguments" msgid "Task Positional Arguments"
msgstr "" msgstr "Positionele argumenten voor taak"
#: documents/models.py:564 #: documents/models.py:564
msgid "JSON representation of the positional arguments used with the task" msgid "JSON representation of the positional arguments used with the task"
msgstr "" msgstr "JSON weergave van de positionele argumenten die gebruikt worden voor de taak"
#: documents/models.py:569 #: documents/models.py:569
msgid "Task Named Arguments" msgid "Task Named Arguments"
msgstr "" msgstr "Argumenten met naam voor taak"
#: documents/models.py:571 #: documents/models.py:571
msgid "JSON representation of the named arguments used with the task" msgid "JSON representation of the named arguments used with the task"
msgstr "" msgstr "JSON weergave van de argumenten met naam die gebruikt worden voor de taak"
#: documents/models.py:578 #: documents/models.py:578
msgid "Task State" msgid "Task State"
msgstr "" msgstr "Taakstatus"
#: documents/models.py:579 #: documents/models.py:579
msgid "Current state of the task being run" msgid "Current state of the task being run"
msgstr "" msgstr "Huidige status van de taak die wordt uitgevoerd"
#: documents/models.py:584 #: documents/models.py:584
msgid "Created DateTime" msgid "Created DateTime"
msgstr "" msgstr "Aangemaakt DateTime"
#: documents/models.py:585 #: documents/models.py:585
msgid "Datetime field when the task result was created in UTC" msgid "Datetime field when the task result was created in UTC"
msgstr "" msgstr "Datetime veld wanneer het resultaat van de taak werd aangemaakt in UTC"
#: documents/models.py:590 #: documents/models.py:590
msgid "Started DateTime" msgid "Started DateTime"
msgstr "" msgstr "Gestart DateTime"
#: documents/models.py:591 #: documents/models.py:591
msgid "Datetime field when the task was started in UTC" msgid "Datetime field when the task was started in UTC"
msgstr "" msgstr "Datetime veld wanneer de taak werd gestart in UTC"
#: documents/models.py:596 #: documents/models.py:596
msgid "Completed DateTime" msgid "Completed DateTime"
msgstr "" msgstr "Voltooid DateTime"
#: documents/models.py:597 #: documents/models.py:597
msgid "Datetime field when the task was completed in UTC" msgid "Datetime field when the task was completed in UTC"
msgstr "" msgstr "Datetime veld wanneer de taak werd voltooid in UTC"
#: documents/models.py:602 #: documents/models.py:602
msgid "Result Data" msgid "Result Data"
msgstr "" msgstr "Resultaatgegevens"
#: documents/models.py:604 #: documents/models.py:604
msgid "The data returned by the task" msgid "The data returned by the task"
msgstr "" msgstr "Gegevens geretourneerd door de taak"
#: documents/models.py:613 #: documents/models.py:613
msgid "Comment for the document" msgid "Comment for the document"
msgstr "" msgstr "Commentaar op het document"
#: documents/models.py:642 #: documents/models.py:642
msgid "comment" msgid "comment"
msgstr "" msgstr "opmerking"
#: documents/models.py:643 #: documents/models.py:643
msgid "comments" msgid "comments"
msgstr "" msgstr "opmerkingen"
#: documents/serialisers.py:72 #: documents/serialisers.py:72
#, python-format #, python-format

View File

@ -109,6 +109,16 @@ def _parse_redis_url(env_redis: Optional[str]) -> Tuple[str]:
def _parse_beat_schedule() -> Dict: def _parse_beat_schedule() -> Dict:
"""
Configures the scheduled tasks, according to default or
environment variables. Task expiration is configured so the task will
expire (and not run), shortly before the default frequency will put another
of the same task into the queue
https://docs.celeryq.dev/en/stable/userguide/periodic-tasks.html#beat-entries
https://docs.celeryq.dev/en/latest/userguide/calling.html#expiration
"""
schedule = {} schedule = {}
tasks = [ tasks = [
{ {
@ -117,6 +127,11 @@ def _parse_beat_schedule() -> Dict:
# Default every ten minutes # Default every ten minutes
"env_default": "*/10 * * * *", "env_default": "*/10 * * * *",
"task": "paperless_mail.tasks.process_mail_accounts", "task": "paperless_mail.tasks.process_mail_accounts",
"options": {
# 1 minute before default schedule sends again
"expires": 9.0
* 60.0,
},
}, },
{ {
"name": "Train the classifier", "name": "Train the classifier",
@ -124,6 +139,11 @@ def _parse_beat_schedule() -> Dict:
# Default hourly at 5 minutes past the hour # Default hourly at 5 minutes past the hour
"env_default": "5 */1 * * *", "env_default": "5 */1 * * *",
"task": "documents.tasks.train_classifier", "task": "documents.tasks.train_classifier",
"options": {
# 1 minute before default schedule sends again
"expires": 59.0
* 60.0,
},
}, },
{ {
"name": "Optimize the index", "name": "Optimize the index",
@ -131,6 +151,12 @@ def _parse_beat_schedule() -> Dict:
# Default daily at midnight # Default daily at midnight
"env_default": "0 0 * * *", "env_default": "0 0 * * *",
"task": "documents.tasks.index_optimize", "task": "documents.tasks.index_optimize",
"options": {
# 1 hour before default schedule sends again
"expires": 23.0
* 60.0
* 60.0,
},
}, },
{ {
"name": "Perform sanity check", "name": "Perform sanity check",
@ -138,6 +164,12 @@ def _parse_beat_schedule() -> Dict:
# Default Sunday at 00:30 # Default Sunday at 00:30
"env_default": "30 0 * * sun", "env_default": "30 0 * * sun",
"task": "documents.tasks.sanity_check", "task": "documents.tasks.sanity_check",
"options": {
# 1 hour before default schedule sends again
"expires": ((7.0 * 24.0) - 1.0)
* 60.0
* 60.0,
},
}, },
] ]
for task in tasks: for task in tasks:
@ -151,9 +183,11 @@ def _parse_beat_schedule() -> Dict:
# - five time-and-date fields # - five time-and-date fields
# - separated by at least one blank # - separated by at least one blank
minute, hour, day_month, month, day_week = value.split(" ") minute, hour, day_month, month, day_week = value.split(" ")
schedule[task["name"]] = { schedule[task["name"]] = {
"task": task["task"], "task": task["task"],
"schedule": crontab(minute, hour, day_week, day_month, month), "schedule": crontab(minute, hour, day_week, day_month, month),
"options": task["options"],
} }
return schedule return schedule
@ -263,6 +297,10 @@ MIDDLEWARE = [
"django.middleware.clickjacking.XFrameOptionsMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware",
] ]
# Optional to enable compression
if __get_boolean("PAPERLESS_ENABLE_COMPRESSION", "yes"): # pragma: nocover
MIDDLEWARE.insert(0, "compression_middleware.middleware.CompressionMiddleware")
ROOT_URLCONF = "paperless.urls" ROOT_URLCONF = "paperless.urls"
FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME") FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME")
@ -280,7 +318,6 @@ _CELERY_REDIS_URL, _CHANNELS_REDIS_URL = _parse_redis_url(
os.getenv("PAPERLESS_REDIS", None), os.getenv("PAPERLESS_REDIS", None),
) )
# TODO: what is this used for?
TEMPLATES = [ TEMPLATES = [
{ {
"BACKEND": "django.template.backends.django.DjangoTemplates", "BACKEND": "django.template.backends.django.DjangoTemplates",
@ -561,22 +598,21 @@ LOGGING = {
# Task queue # # Task queue #
############################################################################### ###############################################################################
TASK_WORKERS = __get_int("PAPERLESS_TASK_WORKERS", 1) # https://docs.celeryq.dev/en/stable/userguide/configuration.html
WORKER_TIMEOUT: Final[int] = __get_int("PAPERLESS_WORKER_TIMEOUT", 1800)
CELERY_BROKER_URL = _CELERY_REDIS_URL CELERY_BROKER_URL = _CELERY_REDIS_URL
CELERY_TIMEZONE = TIME_ZONE CELERY_TIMEZONE = TIME_ZONE
CELERY_WORKER_HIJACK_ROOT_LOGGER = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False
CELERY_WORKER_CONCURRENCY = TASK_WORKERS CELERY_WORKER_CONCURRENCY: Final[int] = __get_int("PAPERLESS_TASK_WORKERS", 1)
TASK_WORKERS = CELERY_WORKER_CONCURRENCY
CELERY_WORKER_MAX_TASKS_PER_CHILD = 1 CELERY_WORKER_MAX_TASKS_PER_CHILD = 1
CELERY_WORKER_SEND_TASK_EVENTS = True CELERY_WORKER_SEND_TASK_EVENTS = True
CELERY_TASK_SEND_SENT_EVENT = True
CELERY_SEND_TASK_SENT_EVENT = True CELERY_SEND_TASK_SENT_EVENT = True
CELERY_TASK_TRACK_STARTED = True CELERY_TASK_TRACK_STARTED = True
CELERY_TASK_TIME_LIMIT = WORKER_TIMEOUT CELERY_TASK_TIME_LIMIT: Final[int] = __get_int("PAPERLESS_WORKER_TIMEOUT", 1800)
CELERY_RESULT_EXTENDED = True CELERY_RESULT_EXTENDED = True
CELERY_RESULT_BACKEND = "django-db" CELERY_RESULT_BACKEND = "django-db"
@ -608,7 +644,7 @@ def default_threads_per_worker(task_workers) -> int:
THREADS_PER_WORKER = os.getenv( THREADS_PER_WORKER = os.getenv(
"PAPERLESS_THREADS_PER_WORKER", "PAPERLESS_THREADS_PER_WORKER",
default_threads_per_worker(TASK_WORKERS), default_threads_per_worker(CELERY_WORKER_CONCURRENCY),
) )
############################################################################### ###############################################################################
@ -637,7 +673,7 @@ CONSUMER_IGNORE_PATTERNS = list(
json.loads( json.loads(
os.getenv( os.getenv(
"PAPERLESS_CONSUMER_IGNORE_PATTERNS", "PAPERLESS_CONSUMER_IGNORE_PATTERNS",
'[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]', # noqa: E501 '[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]', # noqa: E501
), ),
), ),
) )

View File

@ -149,6 +149,11 @@ class TestRedisSocketConversion(TestCase):
class TestCeleryScheduleParsing(TestCase): class TestCeleryScheduleParsing(TestCase):
MAIL_EXPIRE_TIME = 9.0 * 60.0
CLASSIFIER_EXPIRE_TIME = 59.0 * 60.0
INDEX_EXPIRE_TIME = 23.0 * 60.0 * 60.0
SANITY_EXPIRE_TIME = ((7.0 * 24.0) - 1.0) * 60.0 * 60.0
def test_schedule_configuration_default(self): def test_schedule_configuration_default(self):
""" """
GIVEN: GIVEN:
@ -165,18 +170,22 @@ class TestCeleryScheduleParsing(TestCase):
"Check all e-mail accounts": { "Check all e-mail accounts": {
"task": "paperless_mail.tasks.process_mail_accounts", "task": "paperless_mail.tasks.process_mail_accounts",
"schedule": crontab(minute="*/10"), "schedule": crontab(minute="*/10"),
"options": {"expires": self.MAIL_EXPIRE_TIME},
}, },
"Train the classifier": { "Train the classifier": {
"task": "documents.tasks.train_classifier", "task": "documents.tasks.train_classifier",
"schedule": crontab(minute="5", hour="*/1"), "schedule": crontab(minute="5", hour="*/1"),
"options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
}, },
"Optimize the index": { "Optimize the index": {
"task": "documents.tasks.index_optimize", "task": "documents.tasks.index_optimize",
"schedule": crontab(minute=0, hour=0), "schedule": crontab(minute=0, hour=0),
"options": {"expires": self.INDEX_EXPIRE_TIME},
}, },
"Perform sanity check": { "Perform sanity check": {
"task": "documents.tasks.sanity_check", "task": "documents.tasks.sanity_check",
"schedule": crontab(minute=30, hour=0, day_of_week="sun"), "schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
}, },
}, },
schedule, schedule,
@ -203,18 +212,22 @@ class TestCeleryScheduleParsing(TestCase):
"Check all e-mail accounts": { "Check all e-mail accounts": {
"task": "paperless_mail.tasks.process_mail_accounts", "task": "paperless_mail.tasks.process_mail_accounts",
"schedule": crontab(minute="*/50", day_of_week="mon"), "schedule": crontab(minute="*/50", day_of_week="mon"),
"options": {"expires": self.MAIL_EXPIRE_TIME},
}, },
"Train the classifier": { "Train the classifier": {
"task": "documents.tasks.train_classifier", "task": "documents.tasks.train_classifier",
"schedule": crontab(minute="5", hour="*/1"), "schedule": crontab(minute="5", hour="*/1"),
"options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
}, },
"Optimize the index": { "Optimize the index": {
"task": "documents.tasks.index_optimize", "task": "documents.tasks.index_optimize",
"schedule": crontab(minute=0, hour=0), "schedule": crontab(minute=0, hour=0),
"options": {"expires": self.INDEX_EXPIRE_TIME},
}, },
"Perform sanity check": { "Perform sanity check": {
"task": "documents.tasks.sanity_check", "task": "documents.tasks.sanity_check",
"schedule": crontab(minute=30, hour=0, day_of_week="sun"), "schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
}, },
}, },
schedule, schedule,
@ -238,14 +251,17 @@ class TestCeleryScheduleParsing(TestCase):
"Check all e-mail accounts": { "Check all e-mail accounts": {
"task": "paperless_mail.tasks.process_mail_accounts", "task": "paperless_mail.tasks.process_mail_accounts",
"schedule": crontab(minute="*/10"), "schedule": crontab(minute="*/10"),
"options": {"expires": self.MAIL_EXPIRE_TIME},
}, },
"Train the classifier": { "Train the classifier": {
"task": "documents.tasks.train_classifier", "task": "documents.tasks.train_classifier",
"schedule": crontab(minute="5", hour="*/1"), "schedule": crontab(minute="5", hour="*/1"),
"options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
}, },
"Perform sanity check": { "Perform sanity check": {
"task": "documents.tasks.sanity_check", "task": "documents.tasks.sanity_check",
"schedule": crontab(minute=30, hour=0, day_of_week="sun"), "schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
}, },
}, },
schedule, schedule,

View File

@ -14,15 +14,14 @@ TEST_CHANNEL_LAYERS = {
} }
@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
class TestWebSockets(TestCase): class TestWebSockets(TestCase):
@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
async def test_no_auth(self): async def test_no_auth(self):
communicator = WebsocketCommunicator(application, "/ws/status/") communicator = WebsocketCommunicator(application, "/ws/status/")
connected, subprotocol = await communicator.connect() connected, subprotocol = await communicator.connect()
self.assertFalse(connected) self.assertFalse(connected)
await communicator.disconnect() await communicator.disconnect()
@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
@mock.patch("paperless.consumers.StatusConsumer._authenticated") @mock.patch("paperless.consumers.StatusConsumer._authenticated")
async def test_auth(self, _authenticated): async def test_auth(self, _authenticated):
_authenticated.return_value = True _authenticated.return_value = True
@ -33,7 +32,6 @@ class TestWebSockets(TestCase):
await communicator.disconnect() await communicator.disconnect()
@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
@mock.patch("paperless.consumers.StatusConsumer._authenticated") @mock.patch("paperless.consumers.StatusConsumer._authenticated")
async def test_receive(self, _authenticated): async def test_receive(self, _authenticated):
_authenticated.return_value = True _authenticated.return_value = True

View File

@ -12,7 +12,7 @@ class StandardPagination(PageNumberPagination):
class FaviconView(View): class FaviconView(View):
def get(self, request, *args, **kwargs): def get(self, request, *args, **kwargs): # pragma: nocover
favicon = os.path.join( favicon = os.path.join(
os.path.dirname(__file__), os.path.dirname(__file__),
"static", "static",

View File

@ -2,12 +2,13 @@ from django.contrib.auth.models import User
from documents.models import Correspondent from documents.models import Correspondent
from documents.models import DocumentType from documents.models import DocumentType
from documents.models import Tag from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from paperless_mail.models import MailAccount from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule from paperless_mail.models import MailRule
from rest_framework.test import APITestCase from rest_framework.test import APITestCase
class TestAPIMailAccounts(APITestCase): class TestAPIMailAccounts(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/mail_accounts/" ENDPOINT = "/api/mail_accounts/"
def setUp(self): def setUp(self):
@ -165,7 +166,7 @@ class TestAPIMailAccounts(APITestCase):
self.assertEqual(returned_account2.password, "123xyz") self.assertEqual(returned_account2.password, "123xyz")
class TestAPIMailRules(APITestCase): class TestAPIMailRules(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/mail_rules/" ENDPOINT = "/api/mail_rules/"
def setUp(self): def setUp(self):

View File

@ -67,11 +67,6 @@ class TestParserLive(TestCase):
return result return result
# Only run if convert is available
@pytest.mark.skipif(
"PAPERLESS_TEST_SKIP_CONVERT" in os.environ,
reason="PAPERLESS_TEST_SKIP_CONVERT set, skipping Test",
)
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf") @mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
def test_get_thumbnail(self, mock_generate_pdf: mock.MagicMock): def test_get_thumbnail(self, mock_generate_pdf: mock.MagicMock):
""" """
@ -204,11 +199,6 @@ class TestParserLive(TestCase):
"GOTENBERG_LIVE" not in os.environ, "GOTENBERG_LIVE" not in os.environ,
reason="No gotenberg server", reason="No gotenberg server",
) )
# Only run if convert is available
@pytest.mark.skipif(
"PAPERLESS_TEST_SKIP_CONVERT" in os.environ,
reason="PAPERLESS_TEST_SKIP_CONVERT set, skipping Test",
)
def test_generate_pdf_from_mail(self): def test_generate_pdf_from_mail(self):
""" """
GIVEN: GIVEN:
@ -301,11 +291,6 @@ class TestParserLive(TestCase):
"GOTENBERG_LIVE" not in os.environ, "GOTENBERG_LIVE" not in os.environ,
reason="No gotenberg server", reason="No gotenberg server",
) )
# Only run if convert is available
@pytest.mark.skipif(
"PAPERLESS_TEST_SKIP_CONVERT" in os.environ,
reason="PAPERLESS_TEST_SKIP_CONVERT set, skipping Test",
)
def test_generate_pdf_from_html(self): def test_generate_pdf_from_html(self):
""" """
GIVEN: GIVEN:

View File

@ -161,7 +161,7 @@ class RasterisedDocumentParser(DocumentParser):
except Exception: except Exception:
# TODO catch all for various issues with PDFminer.six. # TODO catch all for various issues with PDFminer.six.
# If PDFminer fails, fall back to OCR. # If pdftotext fails, fall back to OCR.
self.log( self.log(
"warning", "warning",
"Error while getting text from PDF document with " "pdfminer.six", "Error while getting text from PDF document with " "pdfminer.six",

View File

@ -364,7 +364,7 @@ class TestParser(DirectoriesMixin, TestCase):
) )
self.assertTrue(os.path.isfile(parser.archive_path)) self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2"]) self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2"])
self.assertFalse("page 3" in parser.get_text().lower()) self.assertNotIn("page 3", parser.get_text().lower())
@override_settings(OCR_PAGES=1, OCR_MODE="force") @override_settings(OCR_PAGES=1, OCR_MODE="force")
def test_multi_page_analog_pages_force(self): def test_multi_page_analog_pages_force(self):
@ -386,8 +386,8 @@ class TestParser(DirectoriesMixin, TestCase):
) )
self.assertTrue(os.path.isfile(parser.archive_path)) self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(parser.get_text().lower(), ["page 1"]) self.assertContainsStrings(parser.get_text().lower(), ["page 1"])
self.assertFalse("page 2" in parser.get_text().lower()) self.assertNotIn("page 2", parser.get_text().lower())
self.assertFalse("page 3" in parser.get_text().lower()) self.assertNotIn("page 3", parser.get_text().lower())
@override_settings(OCR_MODE="skip_noarchive") @override_settings(OCR_MODE="skip_noarchive")
def test_skip_noarchive_withtext(self): def test_skip_noarchive_withtext(self):
@ -660,6 +660,15 @@ class TestParser(DirectoriesMixin, TestCase):
params = parser.construct_ocrmypdf_parameters("", "", "", "") params = parser.construct_ocrmypdf_parameters("", "", "", "")
self.assertNotIn("deskew", params) self.assertNotIn("deskew", params)
with override_settings(OCR_MAX_IMAGE_PIXELS=1_000_001.0):
params = parser.construct_ocrmypdf_parameters("", "", "", "")
self.assertIn("max_image_mpixels", params)
self.assertAlmostEqual(params["max_image_mpixels"], 1, places=4)
with override_settings(OCR_MAX_IMAGE_PIXELS=-1_000_001.0):
params = parser.construct_ocrmypdf_parameters("", "", "", "")
self.assertNotIn("max_image_mpixels", params)
def test_rtl_language_detection(self): def test_rtl_language_detection(self):
""" """
GIVEN: GIVEN:

View File

@ -90,7 +90,7 @@ class TikaDocumentParser(DocumentParser):
with open(document_path, "rb") as document_handle: with open(document_path, "rb") as document_handle:
files = { files = {
"files": ( "files": (
file_name or os.path.basename(document_path), "convert" + os.path.splitext(document_path)[-1],
document_handle, document_handle,
), ),
} }

View File

@ -3,7 +3,9 @@ import os
from pathlib import Path from pathlib import Path
from unittest import mock from unittest import mock
from django.test import override_settings
from django.test import TestCase from django.test import TestCase
from documents.parsers import ParseError
from paperless_tika.parsers import TikaDocumentParser from paperless_tika.parsers import TikaDocumentParser
from requests import Response from requests import Response
@ -54,3 +56,63 @@ class TestTikaParser(TestCase):
self.assertTrue("Creation-Date" in [m["key"] for m in metadata]) self.assertTrue("Creation-Date" in [m["key"] for m in metadata])
self.assertTrue("Some-key" in [m["key"] for m in metadata]) self.assertTrue("Some-key" in [m["key"] for m in metadata])
@mock.patch("paperless_tika.parsers.parser.from_file")
@mock.patch("paperless_tika.parsers.requests.post")
def test_convert_failure(self, post, from_file):
"""
GIVEN:
- Document needs to be converted to PDF
WHEN:
- Gotenberg server returns an error
THEN:
- Parse error is raised
"""
from_file.return_value = {
"content": "the content",
"metadata": {"Creation-Date": "2020-11-21"},
}
response = Response()
response._content = b"PDF document"
response.status_code = 500
post.return_value = response
file = os.path.join(self.parser.tempdir, "input.odt")
Path(file).touch()
with self.assertRaises(ParseError):
self.parser.convert_to_pdf(file, None)
@mock.patch("paperless_tika.parsers.requests.post")
def test_request_pdf_a_format(self, post: mock.Mock):
"""
GIVEN:
- Document needs to be converted to PDF
WHEN:
- Specific PDF/A format requested
THEN:
- Request to Gotenberg contains the expected PDF/A format string
"""
file = os.path.join(self.parser.tempdir, "input.odt")
Path(file).touch()
response = Response()
response._content = b"PDF document"
response.status_code = 200
post.return_value = response
for setting, expected_key in [
("pdfa", "PDF/A-2b"),
("pdfa-2", "PDF/A-2b"),
("pdfa-1", "PDF/A-1a"),
("pdfa-3", "PDF/A-3b"),
]:
with override_settings(OCR_OUTPUT_TYPE=setting):
self.parser.convert_to_pdf(file, None)
post.assert_called_once()
_, kwargs = post.call_args
self.assertEqual(kwargs["data"]["pdfFormat"], expected_key)
post.reset_mock()

View File

@ -7,7 +7,7 @@ max-line-length = 88
[tool:pytest] [tool:pytest]
DJANGO_SETTINGS_MODULE=paperless.settings DJANGO_SETTINGS_MODULE=paperless.settings
addopts = --pythonwarnings=all --cov --cov-report=html --numprocesses auto --quiet addopts = --pythonwarnings=all --cov --cov-report=html --cov-report=xml --numprocesses auto --quiet
env = env =
PAPERLESS_DISABLE_DBHANDLER=true PAPERLESS_DISABLE_DBHANDLER=true