Update ci.yml

This reverts commit aa3d91a3

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,48 @@
+---
+name: Bug report
+about: Something is not working
+title: "[BUG] Concise description of the issue"
+labels: ''
+assignees: ''
+
+---
+
+<!---
+=> Before opening an issue, please check the documentation and see if it helps you resolve your issue: https://paperless-ng.readthedocs.io/en/latest/troubleshooting.html
+=> Please also make sure that you followed the installation instructions.
+=> Please search the issues and look for similar issues before opening a bug report.
+
+=> If you would like to submit a feature request please submit one under https://github.com/jonaswinkler/paperless-ng/discussions/categories/feature-requests
+
+=> If you encounter issues while installing of configuring Paperless-ng, please post that in the "Support" section of the discussions. Remember that Paperless successfully runs on a variety of different systems. If paperless does not start, it's probably an issue with your system, and not an issue of paperless.
+
+=> Don't remove the [BUG] prefix from the title.
+-->
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Webserver logs**
+```
+If available, post any logs from the web server related to your issue.
+```
+
+**Relevant information**
+ - Host OS of the machine running paperless: [e.g. Archlinux / Ubuntu 20.04]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 1.0.0]
+ - Installation method: [docker / bare metal]
+ - Any configuration changes you made in `docker-compose.yml`, `docker-compose.env` or `paperless.conf`.

.github/ISSUE_TEMPLATE/other.md

@@ -0,0 +1,20 @@
+---
+name: Other
+about: Anything that is not a feature request or bug.
+title: "[Other] Title of your issue"
+labels: ''
+assignees: ''
+
+---
+
+<!--
+
+=> Discussions, Feedback and other suggestions belong in the "Discussion" section and not on the issue tracker.
+
+=> If you would like to submit a feature request please submit one under https://github.com/jonaswinkler/paperless-ng/discussions/categories/feature-requests
+
+=> If you encounter issues while installing of configuring Paperless-ng, please post that in the "Support" section of the discussions. Remember that Paperless successfully runs on a variety of different systems. If paperless does not start, it's probably is an issue with your system, and not an issue of paperless.
+
+=> Don't remove the [Other] prefix from the title.
+
+-->

.github/workflows/ansible.yml

@@ -1,19 +1,32 @@
 ---
 name: Ansible Role
 
-on: [push, pull_request]
+on:
+  push:
+    branches-ignore:
+      - 'translations**'
+  pull_request:
+    branches-ignore:
+      - 'translations**'
 
 jobs:
   # https://molecule.readthedocs.io/en/latest/ci.html#github-actions
   test:
     runs-on: ubuntu-latest
-    # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context
-    if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/'))
     steps:
       - name: Check out the codebase
         uses: actions/checkout@v2
+        if: github.event_name != 'pull_request'
         with:
           path: "${{ github.repository }}"
+      - name: Check out the codebase
+        uses: actions/checkout@v2
+        if: github.event_name == 'pull_request'
+        with:
+          # merge commit is not available from tree at this point in time
+          # https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit
+          ref: "${{ github.event.pull_request.head.sha }}"
+          path: "${{ github.repository }}"
       - name: Set up Python
         uses: actions/setup-python@v2
       - name: Set up Docker
@@ -21,15 +34,20 @@ jobs:
       - name: Install dependencies
         run: |
           python3 -m pip install --upgrade pip
-          python3 -m pip install molecule[ansible,docker]
+          python3 -m pip install molecule[ansible,docker] jmespath
           ansible --version
           docker --version
           molecule --version
           python --version
-      - name: Test with molecule
+      - name: Test installation/build/upgrade with molecule
         run: |
           cd ansible
-          molecule test
+          molecule create
+          molecule verify
+          molecule converge
+          molecule idempotence
+          molecule verify
+          molecule destroy
         working-directory: "${{ github.repository }}"
   # # https://galaxy.ansible.com/docs/contributing/importing.html
   # release:

.github/workflows/ci.yml

@@ -1,6 +1,13 @@
 name: ci
 
-on: [push, pull_request]
+on:
+  push:
+    tags: ng-*
+    branches-ignore:
+      - 'translations**'
+  pull_request:
+    branches-ignore:
+      - 'translations**'
 
 jobs:
   documentation:
@@ -28,8 +35,6 @@ jobs:
       -
         name: Install dependencies
         run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev
           pip install --upgrade pipenv
           pipenv install --system --dev --ignore-pipfile
       -
@@ -48,7 +53,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8']
+        python-version: ['3.6', '3.7', '3.8', '3.9']
       fail-fast: false
     steps:
       -
@@ -74,7 +79,7 @@ jobs:
         name: Prepare tests
         run: |
           sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng
           pip install --upgrade pipenv
           pipenv install --system --dev --ignore-pipfile
       -
@@ -107,6 +112,13 @@ jobs:
       uses: actions/setup-node@v2
       with:
         node-version: '15'
+    -
+      name: Configure version on dev branches
+      if: startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev'
+      run: |
+        git_hash=$(git rev-parse --short "$GITHUB_SHA")
+        git_branch=${GITHUB_REF#refs/heads/}
+        sed -i -E "s/version: \"(.*)\"/version: \"${git_branch} ${git_hash}\"/g" src-ui/src/environments/environment.prod.ts
     -
       name: Build frontend
       run: ./compile-frontend.sh
@@ -133,7 +145,7 @@ jobs:
         name: Install dependencies
         run: |
           sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext liblept5
+          sudo apt-get install -qq --no-install-recommends gettext liblept5
           pip3 install -r requirements.txt
       -
         name: Download frontend artifact
@@ -155,6 +167,7 @@ jobs:
           mkdir dist/paperless-ng/scripts
           cp .dockerignore .env Dockerfile Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ng/
           cp paperless.conf.example dist/paperless-ng/paperless.conf
+          cp gunicorn.conf.py dist/paperless-ng/gunicorn.conf.py
           cp docker/ dist/paperless-ng/docker -r
           cp scripts/*.service scripts/*.sh dist/paperless-ng/scripts/
           cp src/ dist/paperless-ng/src -r
@@ -208,7 +221,7 @@ jobs:
           tag_name: ng-${{ steps.get_version.outputs.version }}
           release_name: Paperless-ng ${{ steps.get_version.outputs.version }}
           draft: false
-          prerelease: true
+          prerelease: false
           body: |
             For a complete list of changes, see the changelog at https://paperless-ng.readthedocs.io/en/latest/changelog.html.
       -
@@ -225,7 +238,7 @@ jobs:
 
   # build and push image to docker hub.
   build-docker-image:
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-'))
+    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-'))
     runs-on: ubuntu-latest
     needs: [frontend, tests]
     steps:
@@ -233,15 +246,18 @@ jobs:
         name: Prepare
         id: prepare
         run: |
-          VERSION=edge
+          IMAGE_NAME=jonaswinkler/paperless-ng
           if [[ $GITHUB_REF == refs/tags/ng-* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/ng-}
-          elif [[ $GITHUB_REF == refs/heads/master ]]; then
-            VERSION=latest
+            TAGS=${IMAGE_NAME}:${GITHUB_REF#refs/tags/ng-},${IMAGE_NAME}:latest
+            INSPECT_TAG=${IMAGE_NAME}:latest
           elif [[ $GITHUB_REF == refs/heads/* ]]; then
-            VERSION=${GITHUB_REF#refs/heads/}
+            TAGS=${IMAGE_NAME}:${GITHUB_REF#refs/heads/}
+            INSPECT_TAG=${TAGS}
+          else
+            exit 1
           fi
-          echo ::set-output name=version::${VERSION}
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=inspect_tag::${INSPECT_TAG}
       -
         name: Checkout
         uses: actions/checkout@v2
@@ -279,10 +295,10 @@ jobs:
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm/v7,linux/arm64
           push: true
-          tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }}
+          tags: ${{ steps.prepare.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
       -
         name: Inspect image
         run: |
-          docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }}
+          docker buildx imagetools inspect ${{ steps.prepare.outputs.inspect_tag }}

Dockerfile

@@ -10,10 +10,6 @@ RUN ./configure && make
 
 FROM python:3.7-slim
 
-WORKDIR /usr/src/paperless/
-
-COPY requirements.txt ./
-
 # Binary dependencies
 RUN apt-get update \
   && apt-get -y --no-install-recommends install \
@@ -49,58 +45,61 @@ RUN apt-get update \
 		tesseract-ocr-spa \
 		unpaper \
 		zlib1g \
-		&& rm -rf /var/lib/apt/lists/*
 
 # This pulls in updated dependencies from bullseye to fix some issues with file type detection.
 # TODO: Remove this once bullseye releases.
-RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \
+  && echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \
   && apt-get update \
   && apt-get install --no-install-recommends -y file libmagic-dev \
   && rm -rf /var/lib/apt/lists/* \
   && rm /etc/apt/sources.list.d/bullseye.list
 
-# Python dependencies
-RUN apt-get update \
-  && apt-get -y --no-install-recommends install \
-		build-essential \
-		libpoppler-cpp-dev \
-		libpq-dev \
-		libqpdf-dev \
-	&& python3 -m pip install --upgrade --no-cache-dir supervisor \
-  && python3 -m pip install --no-cache-dir -r requirements.txt \
-	&& apt-get -y purge build-essential libqpdf-dev \
-	&& apt-get -y autoremove --purge \
-	&& rm -rf /var/lib/apt/lists/* \
-	&& mkdir /var/log/supervisord /var/run/supervisord
-
-
-# copy scripts
-# this fixes issues with imagemagick and PDF
-COPY docker/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
-COPY docker/gunicorn.conf.py ./
-COPY docker/supervisord.conf /etc/supervisord.conf
-COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh
-
 # copy jbig2enc
 COPY --from=jbig2enc /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
 COPY --from=jbig2enc /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
 COPY --from=jbig2enc /usr/src/jbig2enc/src/*.h /usr/local/include/
 
+WORKDIR /usr/src/paperless/src/
+
+COPY requirements.txt ../
+
+# Python dependencies
+RUN apt-get update \
+  && apt-get -y --no-install-recommends install \
+		build-essential \
+		libpq-dev \
+		libqpdf-dev \
+	&& python3 -m pip install --upgrade --no-cache-dir supervisor \
+  && python3 -m pip install --no-cache-dir -r ../requirements.txt \
+	&& apt-get -y purge build-essential libqpdf-dev \
+	&& apt-get -y autoremove --purge \
+	&& rm -rf /var/lib/apt/lists/*
+
+# setup docker-specific things
+COPY docker/ ./docker/
+
+RUN cd docker \
+  && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
+	&& mkdir /var/log/supervisord /var/run/supervisord \
+	&& cp supervisord.conf /etc/supervisord.conf \
+	&& cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
+	&& chmod 755 /sbin/docker-entrypoint.sh \
+	&& chmod +x install_management_commands.sh \
+	&& ./install_management_commands.sh \
+	&& cd .. \
+	&& rm docker -rf
+
+COPY gunicorn.conf.py ../
 
 # copy app
-COPY src/ ./src/
+COPY src/ ./
 
 # add users, setup scripts
 RUN addgroup --gid 1000 paperless \
 	&& useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
-	&& chown -R paperless:paperless . \
-	&& chmod 755 /sbin/docker-entrypoint.sh
-
-WORKDIR /usr/src/paperless/src/
-
-RUN sudo -HEu paperless python3 manage.py collectstatic --clear --no-input
-
-RUN sudo -HEu paperless python3 manage.py compilemessages
+	&& chown -R paperless:paperless ../ \
+	&& sudo -HEu paperless python3 manage.py collectstatic --clear --no-input \
+	&& sudo -HEu paperless python3 manage.py compilemessages
 
 VOLUME ["/usr/src/paperless/data", "/usr/src/paperless/media", "/usr/src/paperless/consume", "/usr/src/paperless/export"]
 ENTRYPOINT ["/sbin/docker-entrypoint.sh"]

Pipfile

@@ -17,30 +17,44 @@ django-filter = "~=2.4.0"
 django-q = "~=1.3.4"
 djangorestframework = "~=3.12.2"
 filelock = "*"
-fuzzywuzzy = "*"
+fuzzywuzzy = {extras = ["speedup"], version = "*"}
 gunicorn = "*"
 imap-tools = "*"
 langdetect = "*"
-pdftotext = "*"
+# numpy 1.20.0 drops python 3.6 support
+numpy = "~=1.19.5"
 pathvalidate = "*"
 # pinned to 8.1.0, since aarch64 wheels might not be available beyond that https://github.com/python-pillow/Pillow/issues/5202
 pillow = "==8.1.0"
-pikepdf = "~=2.2.5"
+pikepdf = "~=2.5.0"
 python-gnupg = "*"
 python-dotenv = "*"
 python-dateutil = "*"
-python-Levenshtein = "*"
 python-magic = "*"
 psycopg2-binary = "*"
 redis = "*"
-scikit-learn="~=0.24.0"
+# Pinned because aarch64 wheels and updates cause warnings when loading the classifier model.
+scikit-learn="==0.24.0"
+# Prevent scipy updates because 1.6 is incompatible with python 3.6
+scipy="~=1.5.4"
 whitenoise = "~=5.2.0"
-watchdog = "*"
+watchdog = "~=1.0.0"
 whoosh="~=2.7.4"
 inotifyrecursive = "~=0.3.4"
-ocrmypdf = "~=11.4.5"
+ocrmypdf = "~=11.6"
 tqdm = "*"
 tika = "*"
+# TODO: This will sadly also install daphne+dependencies,
+#  which an ASGI server we don't need. Adds about 15MB image size.
+channels = "~=3.0"
+channels-redis = "*"
+uvicorn = {extras = ["standard"], version = "*"}
+concurrent-log-handler = "*"
+# uvloop 0.15+ incompatible with python 3.6
+uvloop = "~=0.14.0"
+# TODO: keep an eye on piwheel builds and update this once available (https://www.piwheels.org/project/cryptography/)
+cryptography = "~=3.3.2"
+"pdfminer.six" = "*"
 
 [dev-packages]
 coveralls = "*"

README.md

@@ -1,4 +1,5 @@
-![ci](https://github.com/jonaswinkler/paperless-ng/workflows/ci/badge.svg)
+[![ci](https://github.com/jonaswinkler/paperless-ng/workflows/ci/badge.svg)](https://github.com/jonaswinkler/paperless-ng/actions)
+![Ansible Role](https://github.com/jonaswinkler/paperless-ng/workflows/Ansible%20Role/badge.svg)
 [![Documentation Status](https://readthedocs.org/projects/paperless-ng/badge/?version=latest)](https://paperless-ng.readthedocs.io/en/latest/?badge=latest)
 [![Gitter](https://badges.gitter.im/paperless-ng/community.svg)](https://gitter.im/paperless-ng/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Docker Hub Pulls](https://img.shields.io/docker/pulls/jonaswinkler/paperless-ng.svg)](https://hub.docker.com/r/jonaswinkler/paperless-ng)
@@ -6,11 +7,16 @@
 
 # Paperless-ng
 
-[Paperless](https://github.com/the-paperless-project/paperless) is an application by Daniel Quinn and contributors that indexes your scanned documents and allows you to easily search for documents and store metadata alongside your documents.
+[Paperless (click me)](https://github.com/the-paperless-project/paperless) is an application by Daniel Quinn and contributors that indexes your scanned documents and allows you to easily search for documents and store metadata alongside your documents.
 
-Paperless-ng is a fork of the original project, adding a new interface and many other changes under the hood. For a detailed list of changes, have a look at the changelog in the documentation.
+Paperless-ng is a fork of the original project, adding a new interface and many other changes under the hood. These key points should help you decide whether Paperless-ng is something you would prefer over Paperless:
 
-This project is still in development and some things may not work as expected.
+* Interface: The new front end is the main interface for paperless-ng, the old interface still exists but most customizations (such as thumbnails for the document list) have been removed.
+* Encryption: Paperless-ng does not support GnuPG anymore, since storing your data on encrypted file systems (that you optionally mount on demand) achieves about the same result.
+* Resource usage: Paperless-ng does use a bit more resources than Paperless. Running the web server requires about 300MB of RAM or more, depending on the configuration. While adding documents, it requires about 300MB additional RAM, depending on the document. It still runs on Pi (many users do that), but it has been generally geared to better use the resources of more powerful systems.
+* API changes: If you rely on the REST API of paperless, some of its functionality has been changed.
+
+For a detailed list of changes, have a look at the [change log](https://paperless-ng.readthedocs.io/en/latest/changelog.html) in the documentation.
 
 # How it Works
 
@@ -31,8 +37,10 @@ Here's what you get:
 # Features
 
 * Performs OCR on your documents, adds selectable text to image only documents and adds tags, correspondents and document types to your documents.
-* Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and can be configured freely.
-* Single page application front end. Should be pretty snappy. Will be mobile friendly in the future.
+* Supports PDF documents, images, plain text files, and Office documents (Word, Excel, Powerpoint, and LibreOffice equivalents).
+	* Office document support is optional and provided by Apache Tika (see [configuration](https://paperless-ng.readthedocs.io/en/latest/configuration.html#tika-settings))
+* Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and their format can be configured freely.
+* Single page application front end.
 	* Includes a dashboard that shows basic statistics and has document upload.
 	* Filtering by tags, correspondents, types, and more.
 	* Customizable views can be saved and displayed on the dashboard.
@@ -43,48 +51,14 @@ Here's what you get:
 	* Searching for similar documents ("More like this")
 * Email processing: Paperless adds documents from your email accounts.
 	* Configure multiple accounts and filters for each account.
-	* When adding documents from mails, paperless can move these mails to a new folder, mark them as read, flag them or delete them.
+	* When adding documents from mails, paperless can move these mails to a new folder, mark them as read, flag them as important or delete them.
 * Machine learning powered document matching.
 	* Paperless learns from your documents and will be able to automatically assign tags, correspondents and types to documents once you've stored a few documents in paperless.
-* A task processor that processes documents in parallel and also tells you when something goes wrong. On modern multi core systems, consumption is blazing fast.
+* Optimized for multi core systems: Paperless-ng consumes multiple documents in parallel.
+* The integrated sanity checker makes sure that your document archive is in good health.
 
 If you want to see some screenshots of paperless-ng in action, [some are available in the documentation](https://paperless-ng.readthedocs.io/en/latest/screenshots.html).
 
-For a complete list of changes from paperless, check out the [changelog](https://paperless-ng.readthedocs.io/en/latest/changelog.html)
-
-# Roadmap for 1.0
-
-- Make the front end nice (except mobile).
-- Fix whatever bugs I and you find.
-- Make the documentation nice.
-
-## Roadmap for versions beyond 1.0
-
-These are things that I want to add to paperless eventually. They are sorted by priority.
-
-- **More search.** The search backend is incredibly versatile and customizable. Searching is the most important feature of this project and thus, I want to implement things like:
-  - Group and limit search results by correspondent, show “more from this” links in the results.
-- **Nested tags**. Organize tags in a hierarchical structure. This will combine the benefits of folders and tags in one coherent system.
-- **An interactive consumer** that shows its progress for documents it processes on the web page.
-	- With live updates and websockets. This already works on a dev branch, but requires a lot of new dependencies, which I'm not particularly happy about.
-	- Notifications when a document was added with buttons to open the new document right away.
-- **Arbitrary tag colors**. Allow the selection of any color with a color picker.
-
-Apart from that, paperless is pretty much feature complete.
-
-## On the chopping block.
-
-- **GnuPG encrypion.** [Here's a note about encryption in paperless](https://paperless-ng.readthedocs.io/en/latest/administration.html#managing-encryption). The gist of it is that I don't see which attacks this implementation protects against. It gives a false sense of security to users who don't care about how it works.
-
-## Wont-do list.
-
-These features will probably never make it into paperless, since paperless is meant to be an easy to use set-and-forget solution.
-
-- **Document versions.** I might consider adding the ability to update a document with a newer version, but that's about it. The kind of documents that get added to paperless usually don't change at all.
-- **Workflows.** I don't see a use case for these, yet.
-- **Folders.** Tags are superior in just about every way.
-- **Apps / extension support.** Again, paperless is meant to be simple.
-
 # Getting started
 
 The recommended way to deploy paperless is docker-compose. The files in the /docker/hub directory are configured to pull the image from Docker Hub.
@@ -107,13 +81,17 @@ Paperless is currently available in English, German, Dutch and French. Translati
 
 If you want to see paperless in your own language, request that language at transifex and you can start translating after I approve the language.
 
-# Suggestions? Questions? Something not working?
+# Feature Requests
 
-Please open an issue and start a discussion about it!
+Feature requests can be submitted via [GitHub Discussions](https://github.com/jonaswinkler/paperless-ng/discussions/categories/feature-requests), you can search for existing ideas, add your own and vote for the ones you care about! Note that some older feature requests can also be found under [issues](https://github.com/jonaswinkler/paperless-ng/issues).
+
+# Questions? Something not working?
+
+For bugs please [open an issue](https://github.com/jonaswinkler/paperless-ng/issues) or [start a discussion](https://github.com/jonaswinkler/paperless-ng/discussions) if you have questions.
 
 ## Feel like helping out?
 
-There's still lots of things to be done, just have a look at that issue log. If you feel like contributing to the project, please do! Bug fixes and improvements to the front end (I just can't seem to get some of these CSS things right) are always welcome. The documentation has some basic information on how to get started.
+There's still lots of things to be done, just have a look at open issues & discussions. If you feel like contributing to the project, please do! Bug fixes and improvements to the front end (I just can't seem to get some of these CSS things right) are always welcome. The documentation has some basic information on how to get started.
 
 If you want to implement something big: Please start a discussion about that in the issues! Maybe I've already had something similar in mind and we can make it happen together. However, keep in mind that the general roadmap is to make the existing features stable and get them tested. See the roadmap above.
 
@@ -131,4 +109,4 @@ These projects also exist, but their status and compatibility with paperless-ng
 
 # Important Note
 
-Document scanners are typically used to scan sensitive documents.  Things like your social insurance number, tax records, invoices, etc.  Everything is stored in the clear without encryption by default (it needs to be searchable, so if someone has ideas on how to do that on encrypted data, I'm all ears).  This means that Paperless should never be run on an untrusted host.  Instead, I recommend that if you do want to use it, run it locally on a server in your own home.
+Document scanners are typically used to scan sensitive documents.  Things like your social insurance number, tax records, invoices, etc.  Everything is stored in the clear without encryption. This means that Paperless should never be run on an untrusted host.  Instead, I recommend that if you do want to use it, run it locally on a server in your own home.

ansible/README.md

@@ -1,38 +1,114 @@
-Role Name
-=========
+Ansible Role: paperless-ng
+==========================
 
-A brief description of the role goes here.
+Installs and configures paperless-ng EDMS on Debian/Ubuntu systems.
 
 Requirements
 ------------
 
-Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+No special system requirements. Ansible 2.7 or newer is required.
+
+Note that this role requires root access, so either run it in a playbook with a global `become: yes`, or invoke the role in your playbook like:
+
+    - hosts: all
+      roles:
+        - role: ansible
+          become: yes
 
 Role Variables
 --------------
 
-A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+Most configuration variables from paperless-ng itself are available and accept their respective arguments.
+Every `PAPERLESS_*` configuration varaible is lowercased and instead prefixed with `paperlessng_*` in `defaults/main.yml`.
+
+For a full listing including explainations and allowed values, see the current [documentation](https://paperless-ng.readthedocs.io/en/ng-0.9.14/configuration.html).
+
+Additional variables available in this role are listed below, along with default values:
+
+    paperlessng_version: 0.9.14
+
+The [release](https://github.com/jonaswinkler/paperless-ng/releases) archive version of paperless-ng to install.
+
+    paperlessng_redis_host: localhost
+    paperlessng_redis_port: 6379
+
+Seperate configuration values that combine into `PAPERLESS_REDIS`.
+
+    paperlessng_db_type: sqlite
+
+Database to use. Default is file-based SQLite.
+
+    paperlessng_db_host: localhost
+    paperlessng_db_port: 5432
+    paperlessng_db_name: paperlessng
+    paperlessng_db_user: paperlessng
+    paperlessng_db_pass: paperlessng
+    paperlessng_db_sslmode: prefer
+
+Database configuration (only applicable if `paperlessng_db_type == 'postgresql'`).
+
+    paperlessng_directory: /opt/paperless-ng
+
+Root directory paperless-ng is installed into.
+
+    paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv"
+
+Directory used for the virtual environment for paperless-ng.
+
+    paperlessng_ocr_languages:
+      - eng
+
+List of OCR languages to install and configure (`apt search tesseract-ocr-*`).
+
+    paperlessng_use_jbig2enc: True
+
+Whether to install and use [jbig2enc](https://github.com/agl/jbig2enc) for OCRmyPDF.
+
+    paperlessng_big2enc_lossy: False
+
+Whether to use jbig2enc's lossy compression mode.
+
+    paperlessng_superuser_name: paperlessng
+    paperlessng_superuser_email: paperlessng@example.com
+    paperlessng_superuser_password: paperlessng
+
+Credentials of the initial superuser in paperless-ng.
+
+    paperlessng_system_user: paperlessng
+    paperlessng_system_group: paperlessng
+
+System user and group to run the paperless-ng services as (will be created if required).
+
+    paperlessng_listen_address: 127.0.0.1
+    paperlessng_listen_port: 8000
+
+Address and port for the paperless-ng service to listen on.
 
 Dependencies
 ------------
 
-A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+No ansible dependencies.
 
 Example Playbook
 ----------------
+`playbook.yml`:
 
-Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
-
-    - hosts: servers
+    - hosts: all
+      become: yes
+      vars_files:
+        - vars/main.yml
       roles:
-         - { role: username.rolename, x: 42 }
+        - ansible
 
-License
--------
+`vars/main.yml`:
 
-BSD
+    paperlessng_media_root: /mnt/media/smbshare
 
-Author Information
-------------------
+    paperlessng_db_type: postgresql
+    paperlessng_db_pass: PLEASEPROVIDEASTRONGPASSWORDHERE
 
-An optional section for the role authors to include contact information, or a website (HTML is not allowed).
+    paperlessng_secret_key: AGAINPLEASECHANGETHISNOW
+
+    paperlessng_ocr_languages:
+      - eng
+      - deu

ansible/defaults/main.yml

@@ -1,5 +1,19 @@
 ---
-paperlessng_version: 0.9.13
+paperlessng_version: latest  # 'latest', release number, or github branch/tag/commit/ref
+
+# Required services
+paperlessng_redis_host: localhost
+paperlessng_redis_port: 6379
+paperlessng_db_type: sqlite # or postgresql
+# Below entries only apply for paperlessng_db_type=='postgresql'
+paperlessng_db_host: localhost
+paperlessng_db_port: 5432
+paperlessng_db_name: paperlessng
+paperlessng_db_user: paperlessng
+paperlessng_db_pass: paperlessng
+paperlessng_db_sslmode: prefer
+
+# Paths and folders
 paperlessng_directory: /opt/paperless-ng
 paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption"
 paperlessng_data_dir: "{{ paperlessng_directory }}/data"
@@ -8,36 +22,57 @@ paperlessng_static_dir: "{{ paperlessng_directory }}/static"
 paperlessng_filename_format:
 paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv"
 
+# Hosting & Security
+paperlessng_secret_key: PLEASECHANGETHISFORTHELOVEOFGOD
+paperlessng_allowed_hosts: "*"
+paperlessng_cors_allowed_hosts: http://localhost:8000
+paperlessng_force_script_name:
+paperlessng_static_url: /static/
+paperlessng_auto_login_username:
+paperlessng_cookie_prefix: ""
+paperlessng_enable_http_remote_user: False
+
+# OCR settings
 paperlessng_ocr_languages:
   - eng
-paperlessng_time_zone: Europe/Berlin
+paperlessng_ocr_mode: skip
+paperlessng_ocr_output_type: pdfa
+paperlessng_ocr_pages: 0
+paperlessng_ocr_image_dpi:
 # see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr
-paperlessng_ocrmypdf_args:
-  #- "deskew": true  # https://github.com/jonaswinkler/paperless-ng/issues/231
+paperlessng_ocr_user_args:
+  #- "deskew": True  # https://github.com/jonaswinkler/paperless-ng/issues/231
   - "optimize": 1
-paperlessng_use_jbig2enc: true
-paperlessng_big2enc_lossy: false
-paperlessng_tika_enabled: false
+paperlessng_use_jbig2enc: True
+paperlessng_big2enc_lossy: False
+
+# Tika settings
+paperlessng_tika_enabled: False
 paperlessng_tika_endpoint: http://localhost:9998
 paperlessng_tika_gotenberg_endpoint: http://localhost:3000
 
+# Software tweaks
+paperlessng_time_zone: Europe/Berlin
+paperlessng_consumer_polling: 0
+paperlessng_consumer_delete_duplicates: False
+paperlessng_consumer_recursive: False
+paperlessng_consumer_subdirs_as_tags: False
+paperlessng_optimize_thumbnails: True
+paperlessng_post_consume_script:
+paperlessng_filename_date_order:
+paperlessng_filename_parse_transforms:
+paperlessng_thumbnail_font_name: /usr/share/fonts/liberation/LiberationSerif-Regular.ttf
+paperlessng_ignore_dates: ""
+
+# Superuser settings
 paperlessng_superuser_name: paperlessng
 paperlessng_superuser_email: paperlessng@example.com
 paperlessng_superuser_password: paperlessng
 
+# System user settings
 paperlessng_system_user: paperlessng
 paperlessng_system_group: paperlessng
 
+# Webserver settings
 paperlessng_listen_address: 127.0.0.1
 paperlessng_listen_port: 8000
-
-paperlessng_redis_host: localhost
-paperlessng_redis_port: 6379
-
-paperlessng_db_type: sqlite # or postgresql
-# Below entries only apply for paperlessng_db_type=='postgresql'
-paperlessng_db_host: localhost
-paperlessng_db_port: 5432
-paperlessng_db_name: paperlessng
-paperlessng_db_user: paperlessng
-paperlessng_db_pass: paperlessng

ansible/molecule/default/converge.yml

@@ -1,7 +1,10 @@
 ---
-- name: Converge
+- name: update previous release to newest release
   hosts: all
   tasks:
-    - name: "Include ansible"
+    - name: set current github commit as version when available
+      set_fact:
+        paperlessng_version: "{{ lookup('env', 'GITHUB_SHA') | default('master', True) }}"
+    - name: update to newest paperless-ng release
       include_role:
-        name: "ansible"
+        name: ansible

ansible/molecule/default/prepare.yml

@@ -0,0 +1,10 @@
+- name: install previous release
+  hosts: all
+  tasks:
+    - name: set previous version as installation target
+      set_fact:
+        paperlessng_version: latest
+
+    - name: install previous paperless-ng release
+      include_role:
+        name: ansible

ansible/molecule/default/verify.yml

@@ -1,14 +1,94 @@
 ---
-# This is an example playbook to execute Ansible tests.
-
 - name: Verify
   hosts: all
   gather_facts: false
+
+  vars_files:
+    - ../../defaults/main.yml
+
   tasks:
-  - name: check if webserver is up
-    uri:
-      url: http://localhost:8000
-      status_code: [200, 302]
-      return_content: yes
-    register: landingpage
-    failed_when: "'Sign in</button>' not in landingpage.content"
+    - name: check if webserver is up
+      uri:
+        url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}"
+        status_code: [200, 302]
+        return_content: yes
+      register: landingpage
+      failed_when: "'Sign in</button>' not in landingpage.content"
+
+    - name: generate random name and content
+      set_fact:
+        content: "{{ lookup('password', '/dev/null length=65536 chars=ascii_letters') }}"
+        filename: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters') }}"
+
+    - name: check if document posting works
+      uri:
+        url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/post_document/"
+        method: POST
+        body_format: form-multipart
+        body:
+          document:
+            content: "{{ content }}"
+            filename: "{{ filename }}.txt"
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+          Content-Type: text/plain
+        return_content: yes
+      register: post_document
+      failed_when: "'OK' not in post_document.content"
+
+    - name: verify uploaded document has been accepted
+      uri:
+        url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/paperless/"
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "('Consuming ' + filename + '.txt') not in logs.content"
+
+    - name: sleep till consumption finished
+      pause:
+        seconds: 10
+
+    - name: verify uploaded document has been consumed
+      uri:
+        url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/paperless/"
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "filename + ' consumption finished' not in logs.content"
+
+    - name: get documents
+      uri:
+        url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/"
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: documents
+
+    - name: set document index
+      set_fact:
+        index: "{{ documents.json['results'][0]['id'] }}"
+
+    - name: verify uploaded document is avaiable
+      uri:
+        url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/{{ index }}/"
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: document
+      failed_when: "'Not found.' in document.content or content not in document.json['content']"
+
+    - name: check if deleting uploaded document works
+      uri:
+        url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/bulk_edit/"
+        method: POST
+        body_format: json
+        body:
+          documents: ["{{ index }}"]
+          method: delete
+          parameters: {}
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+      register: delete_document
+      failed_when: "'OK' not in delete_document.json['result']"

ansible/tasks/install-release.yml

@@ -0,0 +1,6 @@
+---
+- name: extract paperless-ng
+  unarchive:
+    src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
+    remote_src: yes
+    dest: "{{ tempdir.path }}"

ansible/tasks/install-source.yml

@@ -0,0 +1,112 @@
+---
+- name: install dev dependencies
+  apt:
+    pkg:
+      - git
+      - npm
+      - gettext
+
+- name: create output directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ paperlessng_system_user }}"
+    group: "{{ paperlessng_system_group }}"
+    mode: "750"
+  with_items:
+    - "{{ tempdir.path }}/paperless-ng"
+    - "{{ tempdir.path }}/paperless-ng/scripts"
+
+- block:
+    - name: create temporary git directory
+      tempfile:
+        state: directory
+        path: "{{ paperlessng_directory }}"
+      register: gitdir
+
+    - name: pull paperless-ng
+      git:
+        repo: https://github.com/jonaswinkler/paperless-ng.git
+        dest: "{{ gitdir.path }}"
+        version: "{{ paperlessng_version }}"
+        refspec: "+refs/pull/*:refs/pull/*"
+
+    - name: compile frontend
+      command:
+        cmd: "{{ item }}"
+      args:
+        chdir: "{{ gitdir.path }}/src-ui"
+      failed_when: false
+      with_items:
+        - npm install -g @angular/cli
+        - npm install
+        - ./node_modules/.bin/ng build --prod
+
+    - name: copy application into place
+      copy:
+        src: "{{ gitdir.path }}/{{ item.src }}"
+        remote_src: yes
+        dest: "{{ tempdir.path }}/paperless-ng/{{ item.dest | default('') }}"
+      with_items:
+        - src: CONTRIBUTING.md
+        - src: LICENSE
+        - src: Pipfile
+        - src: Pipfile.lock
+        - src: README.md
+        - src: requirements.txt
+        - src: gunicorn.conf.py
+        - src: paperless.conf.example
+          dest: "paperless.conf"
+
+    - name: glob all scripts
+      find:
+        paths: ["{{ gitdir.path }}/scripts/"]
+        patterns:
+          - "*.service"
+          - "*.sh"
+      register: glob
+
+    - name: copy scripts
+      copy:
+        src: "{{ item.path }}"
+        remote_src: yes
+        dest: "{{ tempdir.path }}/paperless-ng/scripts/"
+      with_items:
+        - "{{ glob.files }}"
+
+    - name: copy sources
+      command:
+        cmd: "cp -r src/ {{ tempdir.path }}/paperless-ng/src"
+      args:
+        chdir: "{{ gitdir.path }}"
+
+    - name: create paperlessng venv
+      command:
+        cmd: "python3 -m virtualenv {{ gitdir.path }}/.venv/ -p /usr/bin/python3"
+
+    - name: install paperlessng requirements
+      command:
+        cmd: "{{ gitdir.path }}/.venv/bin/python3 -m pip install -r {{ gitdir.path }}/requirements.txt"
+
+    - name: compile messages
+      command: "{{ gitdir.path }}/.venv/bin/python3 manage.py compilemessages"
+      args:
+        chdir: "{{ tempdir.path }}/paperless-ng/src/"
+
+    - name: collect static files
+      command: "{{ gitdir.path }}/.venv/bin/python3 manage.py collectstatic --no-input"
+      args:
+        chdir: "{{ tempdir.path }}/paperless-ng/src/"
+
+    - name: remove pycache directories
+      shell: find . -name __pycache__ | xargs rm -r
+      args:
+        chdir: "{{ tempdir.path }}"
+
+    - name: remove temporary git directory
+      file:
+        path: "{{ gitdir.path }}"
+        state: absent
+
+  become: yes
+  become_user: "{{ paperlessng_system_user }}"

ansible/tasks/main.yml

@@ -9,32 +9,37 @@
     update_cache: yes
     pkg:
       # paperless-ng
-      - python3-dev
       - python3-pip
-      - gettext
+      - python3-dev
       - fonts-liberation
       - imagemagick
+      - optipng
+      - gnupg
+      - libpq-dev
+      - libmagic-dev
+      - mime-support
+      # OCRmyPDF
       - unpaper
       - ghostscript
-      - optipng
-      - tesseract-ocr
-      - gnupg
-      - libpoppler-cpp-dev
-      - libmagic-dev
-      - libpq-dev
-      # OCRmyPDF
       - icc-profiles-free
       - qpdf
       - liblept5
       - libxml2
       - pngquant
       - zlib1g
+      - tesseract-ocr
       # dev
       - sudo
       - build-essential
       - python3-setuptools
       - python3-wheel
-      - python3-virtualenv
+
+# upstream virtualenv in Ubuntu 20.04 is broken
+# https://github.com/pypa/virtualenv/issues/1873
+- name: install python virtualenv
+  pip:
+    name: virtualenv
+    extra_args: --upgrade
 
 - name: install ocr languages
   apt:
@@ -97,54 +102,148 @@
     # GNUPG_HOME required due to paperless db.py
     create_home: yes
 
+- block:
+    - name: get latest release version
+      uri:
+        url: https://api.github.com/repos/jonaswinkler/paperless-ng/releases/latest
+        method: GET
+      register: latest_release
+    - name: parse latest release version
+      set_fact:
+        paperlessng_version: "{{ latest_release.json['tag_name'] }}"
+  when: paperlessng_version == "latest"
+
+- name: check if version is ref
+  fail:
+    msg: "Specifying `paperlessng_version` as git ref may not work as expected!"
+  ignore_errors: True  # Output failure (as warning), but don't consider play failed
+  when: paperlessng_version.startswith('refs/')
+
+- block:
+    - name: sanitize version string
+      set_fact:
+        paperlessng_version: "{{ paperlessng_version | regex_replace('^ng-(\\d+\\.\\d+\\.\\d+)$', '\\1') }}"
+    - name: get tag data
+      uri:
+        url: https://api.github.com/repos/jonaswinkler/paperless-ng/tags
+        method: GET
+      register: tags
+    - name: get commit for target tag
+      set_fact:
+        paperlessng_commit: "{{ tags.json | json_query('[?name==`ng-' + paperlessng_version +'`] | [0].commit.sha') }}"
+  when: paperlessng_version | regex_search("^(ng-)?(\d+\.\d+\.\d+)$")
+
+- block:
+    - name: check if version is branch
+      uri:
+        url: "https://api.github.com/repos/jonaswinkler/paperless-ng/branches/{{ paperlessng_version }}"
+        method: GET
+        status_code: [200, 404]
+      register: branch
+    - name: get commit for target branch
+      set_fact:
+        paperlessng_commit: "{{ branch.json | json_query('commit.sha') }}"
+      when: branch.status == 200
+    - block:
+        - name: check if version is commit-or-ref
+          uri:
+            url: "https://api.github.com/repos/jonaswinkler/paperless-ng/commits/{{ paperlessng_version }}"
+            method: GET
+            status_code: [200, 404, 422]
+          register: commit
+        - name: get commit for target commit-or-ref
+          set_fact:
+            paperlessng_commit: "{{ commit.json | json_query('sha') }}"
+          when: commit.status == 200
+        - name: fail
+          fail:
+            msg: "Can not determine commit from `paperlessng_version=={{ paperlessng_version }}`!"
+          when: commit.status != 200
+      when: branch.status == 404
+  when: not(paperlessng_version | regex_search("^(ng-)?(\d+\.\d+\.\d+)$"))
+
 - name: check for paperless-ng installation
   command:
-    cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html'
-  changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+    cmd: "cat {{ paperlessng_directory }}/.installed_version"
+  changed_when: '"No such file or directory" in paperlessng_current_commit.stderr or paperlessng_current_commit.stdout != paperlessng_commit | string'
   failed_when: false
   ignore_errors: yes
-  register: paperlessng_current_version
+  register: paperlessng_current_commit
 
-- name: backup current paperless-ng installation
-  copy:
-    src: "{{ paperlessng_directory }}"
-    remote_src: yes
-    dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/"
-  when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string'
+- name: register current state
+  set_fact:
+    fresh_installation: '{{ "No such file or directory" in paperlessng_current_commit.stderr }}'
+    update_installation: '{{ "No such file or directory" not in paperlessng_current_commit.stderr and paperlessng_current_commit.stdout != paperlessng_commit | string }}'
+    reconfigure_only: "{{ paperlessng_current_commit.stdout == paperlessng_commit | string }}"
 
-- name: create temporary directory
-  tempfile:
-    state: directory
-  register: tempdir
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+- block:
+    - name: backup current paperless-ng installation
+      copy:
+        src: "{{ paperlessng_directory }}"
+        remote_src: yes
+        dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/"
+    - name: remove current paperless sources
+      file:
+        path: "{{ paperlessng_directory }}/{{ item }}"
+        state: absent
+      with_items:
+        - docker
+        - docs
+        - scripts
+        - src
+        - static
+  when: update_installation
 
-- name: extract paperless-ng
-  unarchive:
-    src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
-    remote_src: yes
-    dest: "{{ tempdir.path }}"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
-
-- name: change owner and permissions of paperless-ng
-  command:
-    cmd: "{{ item }}"
-    warn: false
-  with_items:
-  - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
-  - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
-  - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
-
-- name: move paperless-ng
-  command:
-    cmd: "cp -a {{ tempdir.path }}/paperless-ng/ {{ paperlessng_directory }}"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
-
-- name: remove temporary directory
-  file:
-    path: "{{ tempdir.path }}"
-    state: absent
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+- block:
+    - name: create paperless-ng directory and set permissions
+      file:
+        path: "{{ paperlessng_directory }}"
+        state: directory
+        owner: "{{ paperlessng_system_user }}"
+        group: "{{ paperlessng_system_group }}"
+        mode: "750"
+    - name: create temporary directory
+      become: yes
+      become_user: "{{ paperlessng_system_user }}"
+      tempfile:
+        state: directory
+        path: "{{ paperlessng_directory }}"
+      register: tempdir
+    - name: check if version is available as release archive
+      uri:
+        url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
+        method: GET
+        status_code: [200, 404]
+      register: release_archive
+    - name: install paperless-ng from source
+      include_tasks: install-source.yml
+      when: release_archive.status == 404
+    - name: install paperless-ng from release archive
+      include_tasks: install-release.yml
+      when: release_archive.status == 200
+    - name: change owner and permissions of paperless-ng
+      command:
+        cmd: "{{ item }}"
+        warn: false
+      with_items:
+        - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
+        - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
+        - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
+    - name: move paperless-ng
+      command:
+        cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}"
+    - name: store commit hash of installed version
+      copy:
+        content: "{{ paperlessng_commit }}"
+        dest: "{{ paperlessng_directory }}/.installed_version"
+        owner: "{{ paperlessng_system_user }}"
+        group: "{{ paperlessng_system_group }}"
+        mode: "0440"
+    - name: remove temporary directory
+      file:
+        path: "{{ tempdir.path }}"
+        state: absent
+  when: not reconfigure_only
 
 - name: create paperless-ng directories and set permissions
   file:
@@ -154,7 +253,6 @@
     group: "{{ paperlessng_system_group }}"
     mode: "750"
   with_items:
-    - "{{ paperlessng_directory }}"  # ansible `copy:` does not set correct permissions on `dest:` for recursive copies
     - "{{ paperlessng_consumption_dir }}"
     - "{{ paperlessng_data_dir }}"
     - "{{ paperlessng_media_root }}"
@@ -162,7 +260,7 @@
 
 - name: rename initial config
   command:
-    cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template"
+    cmd: "mv -f {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template"
     removes: "{{ paperlessng_directory }}/paperless.conf"
 
 - name: configure paperless-ng
@@ -171,8 +269,10 @@
     regexp: "^#?{{ item.regexp }}="
     line: "{{ item.line }}"
   with_items:
+    # Required services
     - regexp: PAPERLESS_REDIS
       line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}"
+    # Paths and folders
     - regexp: PAPERLESS_CONSUMPTION_DIR
       line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}"
     - regexp: PAPERLESS_DATA_DIR
@@ -183,26 +283,65 @@
       line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}"
     - regexp: PAPERLESS_FILENAME_FORMAT
       line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}"
+    # Hosting & Security
+    - regexp: PAPERLESS_SECRET_KEY
+      line: "PAPERLESS_SECRET_KEY={{ paperlessng_secret_key }}"
+    - regexp: PAPERLESS_ALLOWED_HOSTS
+      line: "PAPERLESS_ALLOWED_HOSTS={{ paperlessng_allowed_hosts }}"
+    - regexp: PAPERLESS_CORS_ALLOWED_HOSTS
+      line: "PAPERLESS_CORS_ALLOWED_HOSTS={{ paperlessng_cors_allowed_hosts }}"
+    - regexp: PAPERLESS_FORCE_SCRIPT_NAME
+      line: "PAPERLESS_FORCE_SCRIPT_NAME={{ paperlessng_force_script_name }}"
+    - regexp: PAPERLESS_STATIC_URL
+      line: "PAPERLESS_STATIC_URL={{ paperlessng_static_url }}"
+    - regexp: PAPERLESS_AUTO_LOGIN_USERNAME
+      line: "PAPERLESS_AUTO_LOGIN_USERNAME={{ paperlessng_auto_login_username }}"
+    - regexp: PAPERLESS_COOKIE_PREFIX
+      line: "PAPERLESS_COOKIE_PREFIX={{ paperlessng_cookie_prefix }}"
+    - regexp: PAPERLESS_ENABLE_HTTP_REMOTE_USER
+      line: "PAPERLESS_ENABLE_HTTP_REMOTE_USER={{ paperlessng_enable_http_remote_user }}"
+    # OCR settings
     - regexp: PAPERLESS_OCR_LANGUAGE
       line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}"
+    - regexp: PAPERLESS_OCR_MODE
+      line: "PAPERLESS_OCR_MODE={{ paperlessng_ocr_mode }}"
+    - regexp: PAPERLESS_OCR_OUTPUT_TYPE
+      line: "PAPERLESS_OCR_OUTPUT_TYPE={{ paperlessng_ocr_output_type }}"
+    - regexp: PAPERLESS_OCR_PAGES
+      line: "PAPERLESS_OCR_PAGES={{ paperlessng_ocr_pages }}"
+    - regexp: PAPERLESS_OCR_IMAGE_DPI
+      line: "PAPERLESS_OCR_IMAGE_DPI={{ paperlessng_ocr_image_dpi }}"
     - regexp: PAPERLESS_OCR_USER_ARGS
-      line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}"
-    - regexp: PAPERLESS_TIME_ZONE
-      line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}"
+      line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocr_user_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}"
+    # Tika settings
     - regexp: PAPERLESS_TIKA_ENABLED
       line: "PAPERLESS_TIKA_ENABLED={{ paperlessng_tika_enabled }}"
-  no_log: yes
-
-- name: configure paperless-ng [tika]
-  lineinfile:
-    path: "{{ paperlessng_directory }}/paperless.conf.template"
-    regexp: "^#?{{ item.regexp }}="
-    line: "'{{ item.line }}' if paperlessng_tika_enabled else '#{{ item.line }}'"
-  with_items:
     - regexp: PAPERLESS_TIKA_ENDPOINT
       line: "PAPERLESS_TIKA_ENDPOINT={{ paperlessng_tika_endpoint }}"
     - regexp: PAPERLESS_TIKA_GOTENBERG_ENDPOINT
       line: "PAPERLESS_TIKA_GOTENBERG_ENDPOINT={{ paperlessng_tika_endpoint }}"
+    # Software tweaks
+    - regexp: PAPERLESS_TIME_ZONE
+      line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}"
+    - regexp: PAPERLESS_CONSUMER_POLLING
+      line: "PAPERLESS_CONSUMER_POLLING={{ paperlessng_consumer_polling }}"
+    - regexp: PAPERLESS_CONSUMER_DELETE_DUPLICATES
+      line: "PAPERLESS_CONSUMER_DELETE_DUPLICATES={{ paperlessng_consumer_delete_duplicates }}"
+    - regexp: PAPERLESS_CONSUMER_RECURSIVE
+      line: "PAPERLESS_CONSUMER_RECURSIVE={{ paperlessng_consumer_recursive }}"
+    - regexp: PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS
+      line: "PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS={{ paperlessng_consumer_subdirs_as_tags }}"
+    - regexp: PAPERLESS_OPTIMIZE_THUMBNAILS
+      line: "PAPERLESS_OPTIMIZE_THUMBNAILS={{ paperlessng_optimize_thumbnails }}"
+    - regexp: PAPERLESS_POST_CONSUME_SCRIPT
+      line: "PAPERLESS_POST_CONSUME_SCRIPT={{ paperlessng_post_consume_script }}"
+    - regexp: PAPERLESS_FILENAME_DATE_ORDER
+      line: "PAPERLESS_FILENAME_DATE_ORDER={{ paperlessng_filename_date_order }}"
+    - regexp: PAPERLESS_THUMBNAIL_FONT_NAME
+      line: "PAPERLESS_THUMBNAIL_FONT_NAME={{ paperlessng_thumbnail_font_name }}"
+    - regexp: PAPERLESS_IGNORE_DATES
+      line: "PAPERLESS_IGNORE_DATES={{ paperlessng_ignore_dates }}"
+  no_log: yes
 
 - name: configure paperless-ng database [sqlite]
   lineinfile:
@@ -228,6 +367,8 @@
       line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}"
     - regexp: PAPERLESS_DBPASS
       line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}"
+    - regexp: PAPERLESS_DBSSLMODE
+      line: "PAPERLESS_DBSSLMODE={{ paperlessng_db_sslmode }}"
   when: paperlessng_db_type == 'postgresql'
   no_log: yes
 
@@ -238,7 +379,7 @@
     dest: /etc/paperless.conf
     owner: root
     group: root
-    mode: '0644'
+    mode: "0644"
   register: configuration
 
 - name: create paperlessng venv
@@ -249,36 +390,21 @@
     creates: "{{ paperlessng_virtualenv }}"
   register: venv
 
-- name: install paperlessng requirements
-  become: yes
-  become_user: "{{ paperlessng_system_user }}"
-  pip:
-    requirements: "{{ paperlessng_directory }}/requirements.txt"
-    executable: "{{ paperlessng_virtualenv }}/bin/pip3"
-    extra_args: --upgrade
-  when: paperlessng_current_version.stdout != paperlessng_version | string
-
-- name: collect static files
-  become: yes
-  become_user: "{{ paperlessng_system_user }}"
-  command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --no-input"
-  when: paperlessng_current_version.stdout != paperlessng_version | string
-  register: static_files
-  changed_when: static_files.stdout is not match("0 static files copied .*")
-
-- name: create database schema
-  become: yes
-  become_user: "{{ paperlessng_system_user }}"
-  command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate"
-  when: paperlessng_current_version.stdout != paperlessng_version | string
-  register: database_schema
-  changed_when: '"No migrations to apply." not in database_schema.stdout'
-
-- name: compile translations
-  become: yes
-  become_user: "{{ paperlessng_system_user }}"
-  command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py compilemessages"
-  when: paperlessng_current_version.stdout != paperlessng_version | string
+- block:
+    - name: install paperlessng requirements
+      become: yes
+      become_user: "{{ paperlessng_system_user }}"
+      pip:
+        requirements: "{{ paperlessng_directory }}/requirements.txt"
+        executable: "{{ paperlessng_virtualenv }}/bin/pip3"
+        extra_args: --upgrade
+    - name: migrate database schema
+      become: yes
+      become_user: "{{ paperlessng_system_user }}"
+      command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate"
+      register: database_schema
+      changed_when: '"No migrations to apply." not in database_schema.stdout'
+  when: not reconfigure_only
 
 - name: configure paperless superuser
   become: yes
@@ -307,7 +433,7 @@
       else:
           User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}')
           print('changed')
-  command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c \"{{ creation_script }}\""
+  command: '{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c "{{ creation_script }}"'
   register: superuser
   changed_when: superuser.stdout == 'changed'
   no_log: yes
@@ -320,7 +446,7 @@
     owner: "{{ paperlessng_system_user }}"
     group: "{{ paperlessng_system_group }}"
     mode: g-w,o-rwx
-  when: venv.changed or paperlessng_current_version.stdout != paperlessng_version | string
+  when: venv.changed or not reconfigure_only
 
 - name: configure ghostscript for PDF
   lineinfile:
@@ -329,6 +455,12 @@
     line: '\1<policy domain="coder" rights="read|write" pattern="PDF" />'
     backrefs: yes
 
+- name: configure gunicorn web server
+  lineinfile:
+    path: "{{ paperlessng_directory }}/gunicorn.conf.py"
+    regexp: '^bind = '
+    line: "bind = '{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}'"
+
 - name: configure systemd services
   ini_file:
     path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}"
@@ -343,34 +475,13 @@
       ]
     - [
         # https://www.freedesktop.org/software/systemd/man/systemd.exec.html
-        {
-          option: "User",
-          value: "{{ paperlessng_system_user }}",
-        },
-        {
-          option: "Group",
-          value: "{{ paperlessng_system_group }}",
-        },
-        {
-          option: "WorkingDirectory",
-          value: "{{ paperlessng_directory }}/src",
-        },
-        {
-          option: "ProtectSystem",
-          value: "full",
-        },
-        {
-          option: "NoNewPrivileges",
-          value: "true",
-        },
-        {
-          option: "PrivateUsers",
-          value: "true",
-        },
-        {
-          option: "PrivateDevices",
-          value: "true",
-        }
+        { option: "User", value: "{{ paperlessng_system_user }}" },
+        { option: "Group", value: "{{ paperlessng_system_group }}" },
+        { option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src" },
+        { option: "ProtectSystem", value: "full" },
+        { option: "NoNewPrivileges", value: "true" },
+        { option: "PrivateUsers", value: "true" },
+        { option: "PrivateDevices", value: "true" },
       ]
 
 - name: configure paperless-consumer service
@@ -392,7 +503,7 @@
     path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service"
     section: "Service"
     option: "ExecStart"
-    value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}"
+    value: "{{ paperlessng_virtualenv }}/bin/gunicorn -c {{ paperlessng_directory }}/gunicorn.conf.py paperless.asgi:application"
 
 - name: copy systemd services
   copy:

crowdin.yml

@@ -0,0 +1,5 @@
+files:
+  - source: /src/locale/en_US/LC_MESSAGES/django.po
+    translation: /src/locale/%locale_with_underscore%/LC_MESSAGES/django.po
+  - source: /src-ui/messages.xlf
+    translation: /src-ui/src/locale/messages.%locale_with_underscore%.xlf

docker/docker-entrypoint.sh

@@ -79,7 +79,11 @@ initialize() {
 		fi
 	done
 
+	echo "creating directory /tmp/paperless"
+	mkdir -p /tmp/paperless
+
 	chown -R paperless:paperless ../
+	chown -R paperless:paperless /tmp/paperless
 
 	migrations
 

docker/install_management_commands.sh

@@ -0,0 +1,6 @@
+for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker;
+do
+	echo "installing $command..."
+	sed "s/management_command/$command/g" management_script.sh > /usr/local/bin/$command
+	chmod +x /usr/local/bin/$command
+done

docker/management_script.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+cd /usr/src/paperless/src/
+
+if [[ $(id -u) == 0 ]] ;
+then
+  sudo -HEu paperless python3 manage.py management_command "$@"
+elif [[ $(id -un) == "paperless" ]] ;
+then
+  python3 manage.py management_command "$@"
+else
+  echo "Unknown user."
+fi

docker/supervisord.conf

@@ -8,7 +8,7 @@ loglevel=info                ; log level; default info; others: debug,warn,trace
 user=root
 
 [program:gunicorn]
-command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.wsgi
+command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.asgi:application
 user=paperless
 
 stdout_logfile=/dev/stdout

docs/administration.rst

@@ -20,6 +20,14 @@ Options available to any installation of paperless:
     metadata to a specific folder. You may import your documents into a
     fresh instance of paperless again or store your documents in another
     DMS with this export.
+*   The document exporter is also able to update an already existing export.
+    Therefore, incremental backups with ``rsync`` are entirely possible.
+
+.. caution::
+
+    You cannot import the export generated with one version of paperless in a
+    different version of paperless. The export contains an exact image of the
+    database, and migrations may change the database layout.
 
 Options available to docker installations:
 
@@ -53,6 +61,9 @@ Restoring
 Updating Paperless
 ##################
 
+Docker Route
+============
+
 If a new release of paperless-ng is available, upgrading depends on how you
 installed paperless-ng in the first place. The releases are available at the
 `release page <https://github.com/jonaswinkler/paperless-ng/releases>`_.
@@ -72,10 +83,10 @@ A.  If you pull the image from the docker hub, all you need to do is:
 
         $ docker-compose pull
         $ docker-compose up
-    
+
     The docker-compose files refer to the ``latest`` version, which is always the latest
     stable release.
-        
+
 B.  If you built the image yourself, do the following:
 
     .. code:: shell-session
@@ -85,64 +96,143 @@ B.  If you built the image yourself, do the following:
         $ docker-compose build
         $ docker-compose up
 
-Running `docker-compose up` will also apply any new database migrations.
+Running ``docker-compose up`` will also apply any new database migrations.
 If you see everything working, press CTRL+C once to gracefully stop paperless.
 Then you can start paperless-ng with ``-d`` to have it run in the background.
 
-Updating paperless without docker
-=================================
+    .. note::
+
+        In version 0.9.14, the update process was changed. In 0.9.13 and earlier, the
+        docker-compose files specified exact versions and pull won't automatically
+        update to newer versions. In order to enable updates as described above, either
+        get the new ``docker-compose.yml`` file from `here <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/compose>`_
+        or edit the ``docker-compose.yml`` file, find the line that says
+
+            .. code::
+
+                image: jonaswinkler/paperless-ng:0.9.x
+
+        and replace the version with ``latest``:
+
+            .. code::
+
+                image: jonaswinkler/paperless-ng:latest
+
+Bare Metal Route
+================
 
 After grabbing the new release and unpacking the contents, do the following:
 
 1.  Update dependencies. New paperless version may require additional
-    dependencies. The dependencies required are listed in the section about 
+    dependencies. The dependencies required are listed in the section about
     :ref:`bare metal installations <setup-bare_metal>`.
 
-2.  Update python requirements. If you use Pipenv, this is done with the following steps.
+2.  Update python requirements. Keep in mind to activate your virtual environment
+    before that, if you use one.
 
     .. code:: shell-session
 
-        $ pip install --upgrade pipenv
-        $ cd /path/to/paperless
-        $ pipenv clean
-        $ pipenv install
+        $ pip install -r requirements.txt
 
-    This creates a new virtual environment (or uses your existing environment)
-    and installs all dependencies into it.
-
-    You can also use the included ``requirements.txt`` file instead and create the virtual
-    environment yourself. This file includes exactly the same dependencies.
-    
 3.  Migrate the database.
 
     .. code:: shell-session
 
         $ cd src
-        $ pipenv run python3 manage.py migrate
-    
+        $ python3 manage.py migrate
+
     This might not actually do anything. Not every new paperless version comes with new
     database migrations.
-    
-        
+
+Ansible Route
+=============
+
+Most of the update process is automated when using the ansible role.
+
+1.  Backup your defined role variables file outside the paperless source-tree:
+
+    .. code:: shell-session
+
+        $ cp ansible/vars.yml ~/vars.yml.old
+
+2.  Pull the release tag you want to update to:
+
+    .. code:: shell-session
+
+        $ git fetch --all
+        $ git checkout ng-0.9.14
+
+3.  Update the role variable definitions ``ansible/vars.yml`` (where appropriate).
+
+4.  Run the ansible playbook you created created during :ref:`installation <setup-ansible>` again:
+
+    .. note::
+
+        When ansible detects that an update run is in progress, it backs up the entire ``paperlessng_directory`` to ``paperlessng_directory-TIMESTAMP``.
+        Updates can be rolled back by simply moving the timestamped folder back to the original location.
+        If the update succeeds and you want to continue using the new release, please don't forget to delete the backup folder.
+
+    .. code:: shell-session
+
+        $ ansible-playbook playbook.yml
+
+
+Downgrading Paperless
+#####################
+
+Downgrades are possible. However, some updates also contain database migrations (these change the layout of the database and may move data).
+In order to move back from a version that applied database migrations, you'll have to revert the database migration *before* downgrading,
+and then downgrade paperless.
+
+This table lists the most recent database migrations for each versions:
+
++---------+-------------------------+
+| Version | Latest migration number |
++---------+-------------------------+
+| 1.0.0   | 1011                    |
++---------+-------------------------+
+| 1.1.0   | 1011                    |
++---------+-------------------------+
+| 1.1.1   | 1012                    |
++---------+-------------------------+
+
+Execute the following management command to migrate your database:
+
+.. code:: shell-session
+
+    $ python3 manage.py migrate documents <migration number>
+
+.. note::
+
+    Some migrations cannot be undone. The command will issue errors if that happens.
+
+.. _utilities-management-commands:
+
 Management utilities
 ####################
 
 Paperless comes with some management commands that perform various maintenance
-tasks on your paperless instance. You can invoke these commands either by
+tasks on your paperless instance. You can invoke these commands in the following way:
+
+With docker-compose, while paperless is running:
 
 .. code:: shell-session
 
     $ cd /path/to/paperless
-    $ docker-compose run --rm webserver <command> <arguments>
+    $ docker-compose exec webserver <command> <arguments>
 
-or
+With docker, while paperless is running:
+
+.. code:: shell-session
+
+    $ docker exec -it <container-name> <command> <arguments>
+
+Bare metal:
 
 .. code:: shell-session
 
     $ cd /path/to/paperless/src
-    $ pipenv run python manage.py <command> <arguments>
-
-depending on whether you use docker or not.
+    $ python3 manage.py <command> <arguments>
 
 All commands have built-in help, which can be accessed by executing them with
 the argument ``--help``.
@@ -157,7 +247,12 @@ backup or migration to another DMS.
 
 .. code::
 
-    document_exporter target
+    document_exporter target [-c] [-f] [-d]
+
+    optional arguments:
+    -c, --compare-checksums
+    -f, --use-filename-format
+    -d, --delete
 
 ``target`` is a folder to which the data gets written. This includes documents,
 thumbnails and a ``manifest.json`` file. The manifest contains all metadata from
@@ -167,6 +262,24 @@ When you use the provided docker compose script, specify ``../export`` as the
 target. This path inside the container is automatically mounted on your host on
 the folder ``export``.
 
+If the target directory already exists and contains files, paperless will assume
+that the contents of the export directory are a previous export and will attempt
+to update the previous export. Paperless will only export changed and added files.
+Paperless determines whether a file has changed by inspecting the file attributes
+"date/time modified" and "size". If that does not work out for you, specify
+``--compare-checksums`` and paperless will attempt to compare file checksums instead.
+This is slower.
+
+Paperless will not remove any existing files in the export directory. If you want
+paperless to also remove files that do not belong to the current export such as files
+from deleted documents, specify ``--delete``. Be careful when pointing paperless to
+a directory that already contains other files.
+
+The filenames generated by this command follow the format
+``[date created] [correspondent] [title].[extension]``.
+If you want paperless to use ``PAPERLESS_FILENAME_FORMAT`` for exported filenames
+instead, specify ``--use-filename-format``.
+
 
 .. _utilities-importer:
 
@@ -297,6 +410,34 @@ the naming scheme.
 The command takes no arguments and processes all your documents at once.
 
 
+.. _utilities-sanity-checker:
+
+Sanity checker
+==============
+
+Paperless has a built-in sanity checker that inspects your document collection for issues.
+
+The issues detected by the sanity checker are as follows:
+
+* Missing original files.
+* Missing archive files.
+* Inaccessible original files due to improper permissions.
+* Inaccessible archive files due to improper permissions.
+* Corrupted original documents by comparing their checksum against what is stored in the database.
+* Corrupted archive documents by comparing their checksum against what is stored in the database.
+* Missing thumbnails.
+* Inaccessible thumbnails due to improper permissions.
+* Documents without any content (warning).
+* Orphaned files in the media directory (warning). These are files that are not referenced by any document im paperless.
+
+
+.. code::
+
+    document_sanity_checker
+
+The command takes no arguments. Depending on the size of your document archive, this may take some time.
+
+
 Fetching e-mail
 ===============
 
@@ -361,7 +502,7 @@ Documents can be stored in Paperless using GnuPG encryption.
     Furthermore, the entire text content of the documents is stored plain in the
     database, even if your documents are encrypted. Filenames are not encrypted as
     well.
-    
+
     Also, the web server provides transparent access to your encrypted documents.
 
     Consider running paperless on an encrypted filesystem instead, which will then
@@ -384,6 +525,3 @@ Basic usage to disable encryption of your document store:
 .. code::
 
     decrypt_documents [--passphrase SECR3TP4SSPHRA$E]
-
-
-.. _Pipenv: https://pipenv.pypa.io/en/latest/

docs/advanced_usage.rst

@@ -217,6 +217,7 @@ will create a directory structure as follows:
 
 Paperless provides the following placeholders withing filenames:
 
+* ``{asn}``: The archive serial number of the document, or "none".
 * ``{correspondent}``: The name of the correspondent, or "none".
 * ``{document_type}``: The name of the document type, or "none".
 * ``{tag_list}``: A comma separated list of all tags assigned to the document.

docs/changelog.rst

@@ -5,6 +5,178 @@
 Changelog
 *********
 
+paperless-ng 1.2.0
+##################
+
+* Changes to the OCRmyPDF integration
+
+  * Added support for deskewing and automatic rotation of incorrectly rotated pages. This is enabled by default, see :ref:`configuration-ocr`.
+  * Better support for encrypted files.
+  * Better support for various other PDF files: Paperless will now attempt to force OCR with safe options when OCR fails with the configured options.
+  * Added an explicit option to skip cleaning with ``unpaper``.
+
+* Download multiple selected documents as a zip archive.
+
+* The document list now remembers the current page.
+
+* Improved responsiveness when switching between saved views and the document list.
+
+* Increased the default wait time when observing files in the consumption folder
+  with polling from 1 to 5 seconds. This will decrease the likelihood of paperless
+  consuming partially written files.
+
+* Fixed a crash of the document archiver management command when trying to process documents with unknown mime types.
+
+* Paperless no longer depends on ``libpoppler-cpp-dev``.
+
+.. note::
+
+  Some packages that paperless depends on are slowly dropping Python 3.6
+  support one after another, including the web server. Supporting Python
+  3.6 means that I cannot update these packages anymore.
+
+  At some point, paperless will drop Python 3.6 support. If using a bare
+  metal installation and you're still on Python 3.6, upgrade to 3.7 or newer.
+
+  If using docker, this does not affect you.
+
+paperless-ng 1.1.4
+##################
+
+* Added English (GB) locale.
+
+* Added ISO-8601 date display option.
+
+paperless-ng 1.1.3
+##################
+
+* Added a docker-specific configuration option to adjust the number of
+  worker processes of the web server. See :ref:`configuration-docker`.
+
+* Some more memory usage optimizations.
+
+* Don't show inbox statistics if no inbox tag is defined.
+
+paperless-ng 1.1.2
+##################
+
+* Always show top left corner of thumbnails, even for extra wide documents.
+
+* Added a management command for executing the sanity checker directly.
+  See :ref:`utilities-sanity-checker`.
+
+* The weekly sanity check now reports messages in the log files.
+
+* Fixed an issue with the metadata tab not reporting anything in case of missing files.
+
+* Reverted a change from 1.1.0 that caused huge memory usage due to redis caching.
+
+* Some memory usage optimizations.
+
+paperless-ng 1.1.1
+##################
+
+This release contains new database migrations.
+
+* Fixed a bug in the sanity checker that would cause it to display "x not in list" errors instead of actual issues.
+
+* Fixed a bug with filename generation for archive filenames that would cause the archive files of two documents to overlap.
+
+  * This happened when ``PAPERLESS_FILENAME_FORMAT`` is used and the filenames of two or more documents are the same, except for the file extension.
+  * Paperless will now store the archive filename in the database as well instead of deriving it from the original filename, and use the
+    same logic for detecting and avoiding filename clashes that's also used for original filenames.
+  * The migrations will repair any missing archive files. If you're using tika, ensure that tika is running while performing the migration. Docker-compose will take care of that.
+
+* Fixed a bug with thumbnail regeneration when TIKA integration was used.
+
+* Added ASN as a placeholder field to the filename format.
+
+* The docker image now comes with built-in shortcuts for most management commands. These are now the recommended way to execute management commands, since these
+  also ensure that they're always executed as the paperless user and you're less likely to run into permission issues. See :ref:`utilities-management-commands`.
+
+paperless-ng 1.1.0
+##################
+
+* Document processing status
+
+  * Paperless now shows the status of processing documents on the dashboard in real time.
+  * Status notifications when
+
+    * New documents are detected in the consumption folder, in mails, uploaded on the front end,
+      or added with one of the mobile apps.
+    * Documents are successfully added to paperless.
+    * Document consumption failed (with error messages)
+
+  * Configuration options to enable/disable individual notifications.
+
+* Live updates to document lists and saved views when new documents are added.
+
+  .. hint::
+
+    For status notifications and live updates to work, paperless now requires an `ASGI <https://asgi.readthedocs.io/en/latest/>`_-enabled
+    web server. The docker images uses ``gunicorn`` and an ASGI-enabled worker called `uvicorn <http://www.uvicorn.org/>`_,
+    and there is no need to configure anything.
+
+    For bare metal installations, changes are required for the notifications to work. Adapt the service ``paperless-webserver.service``
+    to use the supplied ``gunicorn.conf.py`` configuration file and adapt the reference to the ASGI application as follows:
+
+    .. code::
+
+      ExecStart=/opt/paperless/.local/bin/gunicorn -c /opt/paperless/gunicorn.conf.py paperless.asgi:application
+
+    Paperless will continue to work with WSGI, but you will not get any status notifications.
+
+    Apache ``mod_wsgi`` users, see :ref:`this note <faq-mod_wsgi>`.
+
+* Paperless now offers suggestions for tags, correspondents and types on the document detail page.
+
+* Added an interactive easy install script that automatically downloads, configures and starts paperless with docker.
+
+* Official support for Python 3.9.
+
+* Other changes and fixes
+
+  * Adjusted the default parallelization settings to run more than one task in parallel on systems with 4 or less cores.
+    This addresses issues with paperless not consuming any new files when other tasks are running.
+
+  * Fixed a rare race condition that would cause paperless to process incompletely written files when using the upload on the dashboard.
+
+  * The document classifier no longer issues warnings and errors when auto matching is not used at all.
+
+  * Better icon for document previews.
+
+  * Better info section in the side bar.
+
+  * Paperless no longer logs to the database. Instead, logs are written to rotating log files. This solves many "database is locked"
+    issues on Raspberry Pi, especially when SQLite is used.
+
+  * By default, log files are written to ``PAPERLESS_DATA_DIR/log/``. Logging settings can be adjusted with
+    ``PAPERLESS_LOGGING_DIR``, ``PAPERLESS_LOGROTATE_MAX_SIZE`` and
+    ``PAPERLESS_LOGROTATE_MAX_BACKUPS``.
+
+paperless-ng 1.0.0
+##################
+
+Nothing special about this release, but since there are relatively few bug reports coming in, I think that this is reasonably stable.
+
+* Document export
+
+  * The document exporter has been rewritten to support updating an already existing export in place.
+    This enables incremental backups with ``rsync``.
+  * The document exporter supports naming exported files according to ``PAPERLESS_FILENAME_FORMAT``.
+  * The document exporter locks the media directory and the database during execution to ensure that
+    the resulting export is consistent.
+  * See the :ref:`updated documentation <utilities-exporter>` for more details.
+
+* Other changes and additions
+
+  * Added a language selector to the settings.
+  * Added date format options to the settings.
+  * Range selection with shift clicking is now possible in the document list.
+  * Filtering correspondent, type and tag management pages by name.
+  * Focus "Name" field in dialogs by default.
+
+
 paperless-ng 0.9.14
 ###################
 
@@ -116,7 +288,7 @@ paperless-ng 0.9.10
   * There are some configuration options in the settings to alter the behavior.
 
 * Other changes and additions
-  
+
   * Thanks to `zjean`_, paperless now publishes a webmanifest, which is useful for adding the application to home screens on mobile devices.
   * The Paperless-ng logo now navigates to the dashboard.
   * Filter for documents that don't have any correspondents, types or tags assigned.
@@ -136,7 +308,7 @@ paperless-ng 0.9.10
   The bulk delete operations did not update the search index. Therefore, documents that you deleted remained in the index and
   caused the search to return messages about missing documents when searching. Further bulk operations will properly update
   the index.
-  
+
   However, this change is not retroactive: If you used the delete method of the bulk editor, you need to reindex your search index
   by :ref:`running the management command document_index with the argument reindex <administration-index>`.
 
@@ -191,12 +363,12 @@ paperless-ng 0.9.7
 
   * Thanks to the hard work of `Michael Shamoon`_, paperless now comes with a much more streamlined UI for
     filtering documents.
-  
+
   * `Michael Shamoon`_ replaced the document preview with another component. This should fix compatibility with Safari browsers.
 
   * Added buttons to the management pages to quickly show all documents with one specific tag, correspondent, or title.
-  
-  * Paperless now stores your saved views on the server and associates them with your user account. 
+
+  * Paperless now stores your saved views on the server and associates them with your user account.
     This means that you can access your views on multiple devices and have separate views for different users.
     You will have to recreate your views.
 
@@ -214,7 +386,7 @@ paperless-ng 0.9.7
     This option enables you to be logged in into multiple instances by specifying different cookie names for each instance.
 
 * Fixes
-  
+
   * Sometimes paperless would assign dates in the future to newly consumed documents.
   * The filename format fields ``{created_month}`` and ``{created_day}`` now use a leading zero for single digit values.
   * The filename format field ``{tags}`` can no longer be used without arguments.

docs/conf.py

@@ -28,7 +28,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Paperless-ng'
-copyright = u'2015, Daniel Quinn'
+copyright = u'2021, Daniel Quinn, Jonas Winkler'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

docs/configuration.rst

@@ -72,13 +72,13 @@ PAPERLESS_CONSUMPTION_DIR=<path>
     container. Change the local consumption directory in the docker-compose.yml
     file instead.
 
-    Defaults to "../consume", relative to the "src" directory.
+    Defaults to "../consume/", relative to the "src" directory.
 
 PAPERLESS_DATA_DIR=<path>
     This is where paperless stores all its data (search index, SQLite database,
     classification model, etc).
 
-    Defaults to "../data", relative to the "src" directory.
+    Defaults to "../data/", relative to the "src" directory.
 
 PAPERLESS_MEDIA_ROOT=<path>
     This is where your documents and thumbnails are stored.
@@ -86,7 +86,7 @@ PAPERLESS_MEDIA_ROOT=<path>
     You can set this and PAPERLESS_DATA_DIR to the same folder to have paperless
     store all its data within the same volume.
 
-    Defaults to "../media", relative to the "src" directory.
+    Defaults to "../media/", relative to the "src" directory.
 
 PAPERLESS_STATICDIR=<path>
     Override the default STATIC_ROOT here.  This is where all static files
@@ -94,7 +94,7 @@ PAPERLESS_STATICDIR=<path>
 
     Unless you're doing something fancy, there is no need to override this.
 
-    Defaults to "../static", relative to the "src" directory.
+    Defaults to "../static/", relative to the "src" directory.
 
 PAPERLESS_FILENAME_FORMAT=<format>
     Changes the filenames paperless uses to store documents in the media directory.
@@ -102,6 +102,25 @@ PAPERLESS_FILENAME_FORMAT=<format>
 
     Default is none, which disables this feature.
 
+PAPERLESS_LOGGING_DIR=<path>
+    This is where paperless will store log files.
+
+    Defaults to "``PAPERLESS_DATA_DIR``/log/".
+
+
+Logging
+#######
+
+PAPERLESS_LOGROTATE_MAX_SIZE=<num>
+    Maximum file size for log files before they are rotated, in bytes.
+
+    Defaults to 1 MiB.
+
+PAPERLESS_LOGROTATE_MAX_BACKUPS=<num>
+    Number of rotated log files to keep.
+
+    Defaults to 20.
+
 Hosting & Security
 ##################
 
@@ -183,7 +202,6 @@ Paperless uses `OCRmyPDF <https://ocrmypdf.readthedocs.io/en/latest/>`_ for
 performing OCR on documents and images. Paperless uses sensible defaults for
 most settings, but all of them can be configured to your needs.
 
-
 PAPERLESS_OCR_LANGUAGE=<lang>
     Customize the language that paperless will attempt to use when
     parsing documents.
@@ -226,6 +244,54 @@ PAPERLESS_OCR_MODE=<mode>
     The default is ``skip``, which only performs OCR when necessary and always
     creates archived documents.
 
+    Read more about this in the `OCRmyPDF documentation <https://ocrmypdf.readthedocs.io/en/latest/advanced.html#when-ocr-is-skipped>`_.
+
+PAPERLESS_OCR_CLEAN=<mode>
+    Tells paperless to use ``unpaper`` to clean any input document before
+    sending it to tesseract. This uses more resources, but generally results
+    in better OCR results. The following modes are available:
+
+    *   ``clean``: Apply unpaper.
+    *   ``clean-final``: Apply unpaper, and use the cleaned images to build the
+        output file instead of the original images.
+    *   ``none``: Do not apply unpaper.
+
+    Defaults to ``clean``.
+
+    .. note::
+
+        ``clean-final`` is incompatible with ocr mode ``redo``. When both
+        ``clean-final`` and the ocr mode ``redo`` is configured, ``clean``
+        is used instead.
+
+PAPERLESS_OCR_DESKEW=<bool>
+    Tells paperless to correct skewing (slight rotation of input images mainly
+    due to improper scanning)
+
+    Defaults to ``true``, which enables this feature.
+
+    .. note::
+
+        Deskewing is incompatible with ocr mode ``redo``. Deskewing will get
+        disabled automatically if ``redo`` is used as the ocr mode.
+
+PAPERLESS_OCR_ROTATE_PAGES=<bool>
+    Tells paperless to correct page rotation (90°, 180° and 270° rotation).
+
+    If you notice that paperless is not rotating pages incorrectly rotated
+    pages (or vice versa), try adjusting the threshold up or down (see below).
+
+    Defaults to ``true``, which enables this feature.
+
+
+PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=<num>
+    Adjust the threshold for automatic page rotation by ``PAPERLESS_OCR_ROTATE_PAGES``.
+    This is an arbitrary value reported by tesseract. "15" is a very conservative value,
+    whereas "2" is a very aggressive option and will often result correctly rotated pages
+    being rotated as well.
+
+    Defaults to "12".
+
 PAPERLESS_OCR_OUTPUT_TYPE=<type>
     Specify the the type of PDF documents that paperless should produce.
 
@@ -252,7 +318,6 @@ PAPERLESS_OCR_PAGES=<num>
 
     Defaults to 0, which disables this feature and always uses all pages.
 
-
 PAPERLESS_OCR_IMAGE_DPI=<num>
     Paperless will OCR any images you put into the system and convert them
     into PDF documents. This is useful if your scanner produces images.
@@ -263,11 +328,11 @@ PAPERLESS_OCR_IMAGE_DPI=<num>
 
     Set this to the DPI your scanner produces images at.
 
-    Default is none, which causes paperless to fail if no DPI information is
-    present in an image.
+    Default is none, which will automatically calculate image DPI so that
+    the produced PDF documents are A4 sized.
 
 
-PAPERLESS_OCR_USER_ARG=<json>
+PAPERLESS_OCR_USER_ARGS=<json>
     OCRmyPDF offers many more options. Use this parameter to specify any
     additional arguments you wish to pass to OCRmyPDF. Since Paperless uses
     the API of OCRmyPDF, you have to specify these in a format that can be
@@ -333,7 +398,7 @@ requires are as follows:
                 PAPERLESS_TIKA_ENABLED: 1
                 PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
                 PAPERLESS_TIKA_ENDPOINT: http://tika:9998
-        
+
         # ...
 
         gotenberg:
@@ -376,25 +441,24 @@ PAPERLESS_THREADS_PER_WORKER=<num>
         use a higher thread per worker count.
 
     The default is a balance between the two, according to your CPU core count,
-    with a slight favor towards threads per worker, and leaving at least one core
-    free for other tasks:
+    with a slight favor towards threads per worker:
 
     +----------------+---------+---------+
     | CPU core count | Workers | Threads |
     +----------------+---------+---------+
     |              1 |       1 |       1 |
     +----------------+---------+---------+
-    |              2 |       1 |       1 |
+    |              2 |       2 |       1 |
     +----------------+---------+---------+
-    |              4 |       1 |       3 |
+    |              4 |       2 |       2 |
     +----------------+---------+---------+
-    |              6 |       2 |       2 |
+    |              6 |       2 |       3 |
     +----------------+---------+---------+
-    |              8 |       2 |       3 |
+    |              8 |       2 |       4 |
     +----------------+---------+---------+
-    |             12 |       3 |       3 |
+    |             12 |       3 |       4 |
     +----------------+---------+---------+
-    |             16 |       3 |       5 |
+    |             16 |       4 |       4 |
     +----------------+---------+---------+
 
     If you only specify PAPERLESS_TASK_WORKERS, paperless will adjust
@@ -537,3 +601,65 @@ PAPERLESS_GS_BINARY=<path>
 
 PAPERLESS_OPTIPNG_BINARY=<path>
     Defaults to "/usr/bin/optipng".
+
+
+.. _configuration-docker:
+
+Docker-specific options
+#######################
+
+These options don't have any effect in ``paperless.conf``. These options adjust
+the behavior of the docker container. Configure these in `docker-compose.env`.
+
+PAPERLESS_WEBSERVER_WORKERS=<num>
+    The number of worker processes the webserver should spawn. More worker processes
+    usually result in the front end to load data much quicker. However, each worker process
+    also loads the entire application into memory separately, so increasing this value
+    will increase RAM usage.
+
+    Consider configuring this to 1 on low power devices with limited amount of RAM.
+
+    Defaults to 2.
+
+USERMAP_UID=<uid>
+    The ID of the paperless user in the container. Set this to your actual user ID on the
+    host system, which you can get by executing
+
+    .. code:: shell-session
+
+        $ id -u
+
+    Paperless will change ownership on its folders to this user, so you need to get this right
+    in order to be able to write to the consumption directory.
+
+    Defaults to 1000.
+
+USERMAP_GID=<gid>
+    The ID of the paperless Group in the container. Set this to your actual group ID on the
+    host system, which you can get by executing
+
+    .. code:: shell-session
+
+        $ id -g
+
+    Paperless will change ownership on its folders to this group, so you need to get this right
+    in order to be able to write to the consumption directory.
+
+    Defaults to 1000.
+
+PAPERLESS_OCR_LANGUAGES=<list>
+    Additional OCR languages to install. By default, paperless comes with
+    English, German, Italian, Spanish and French. If your language is not in this list, install
+    additional languages with this configuration option:
+
+    .. code:: bash
+
+        PAPERLESS_OCR_LANGUAGES=tur ces
+
+    To actually use these languages, also set the default OCR language of paperless:
+
+    .. code:: bash
+
+        PAPERLESS_OCR_LANGUAGE=tur
+
+    Defaults to none, which does not install any additional languages.

docs/extending.rst

@@ -109,6 +109,30 @@ This will build the front end and put it in a location from which the Django ser
 it as static content. This way, you can verify that authentication is working.
 
 
+Building the documentation
+==========================
+
+The documentation is built using sphinx. I've configured ReadTheDocs to automatically build
+the documentation when changes are pushed. If you want to build the documentation locally,
+this is how you do it:
+
+1.  Install python dependencies.
+
+    .. code:: shell-session
+
+        $ cd /path/to/paperless
+        $ pipenv install --dev
+
+2.  Build the documentation
+
+    .. code:: shell-session
+
+        $ cd /path/to/paperless/docs
+        $ pipenv run make clean html
+
+This will build the HTML documentation, and put the resulting files in the ``_build/html``
+directory.
+
 Extending Paperless
 ===================
 

docs/faq.rst

@@ -62,9 +62,9 @@ file extensions do not matter.
 
 **A:** The short answer is yes. I've tested it on a Raspberry Pi 3 B.
 The long answer is that certain parts of
-Paperless will run very slow, such as the tesseract OCR. On Raspberry Pi,
+Paperless will run very slow, such as the OCR. On Raspberry Pi,
 try to OCR documents before feeding them into paperless so that paperless can
-reuse the text. The web interface should be a lot snappier, since it runs
+reuse the text. The web interface is a lot snappier, since it runs
 in your browser and paperless has to do much less work to serve the data.
 
 .. note::
@@ -76,7 +76,14 @@ in your browser and paperless has to do much less work to serve the data.
 **Q:** *How do I install paperless-ng on Raspberry Pi?*
 
 **A:** Docker images are available for arm and arm64 hardware, so just follow
-the docker-compose instructions, or go the bare metal route.
+the docker-compose instructions. Apart from more required disk space compared to
+a bare metal installation, docker comes with close to zero overhead, even on
+Raspberry Pi.
+
+If you decide to got with the bare metal route, be aware that some of the
+python requirements do not have precompiled packages for ARM / ARM64. Installation
+of these will require additional development libraries and compilation will take
+a long time.
 
 **Q:** *How do I run this on unRaid?*
 
@@ -95,12 +102,21 @@ occasionally build the image on and see if it works.
 
 **Q:** *How do I proxy this with NGINX?*
 
-.. code::
+**A:** See :ref:`here <setup-nginx>`.
 
-    location / {
-        proxy_pass http://localhost:8000/
-    }
+.. _faq-mod_wsgi:
 
-And that's about it. Paperless serves everything, including static files by itself
-when running the docker image. If you want to do anything fancy, you have to
-install paperless bare metal.
+**Q:** *How do I get WebSocket support with Apache mod_wsgi*?
+
+**A:** ``mod_wsgi`` by itself does not support ASGI. Paperless will continue
+to work with WSGI, but certain features such as status notifications about
+document consumption won't be available.
+
+If you want to continue using ``mod_wsgi``, you will have to run an ASGI-enabled
+web server as well that processes WebSocket connections, and configure Apache to
+redirect WebSocket connections to this server. Multiple options for ASGI servers
+exist:
+
+* ``gunicorn`` with ``uvicorn`` as the worker implementation (the default of paperless)
+* ``daphne`` as a standalone server, which is the reference implementation for ASGI.
+* ``uvicorn`` as a standalone server

docs/index.rst

@@ -70,8 +70,8 @@ Contents
    configuration
    api
    faq
-   extending
    troubleshooting
+   extending
    contributing
    scanners
    screenshots

docs/setup.rst

@@ -20,45 +20,45 @@ Paperless consists of the following components:
     .. code:: shell-session
 
         $ cd /path/to/paperless/src/
-        $ pipenv run gunicorn -c /usr/src/paperless/gunicorn.conf.py -b 0.0.0.0:8000 paperless.wsgi
+        $ gunicorn -c ../gunicorn.conf.py paperless.wsgi
 
     or by any other means such as Apache ``mod_wsgi``.
 
 *   **The consumer:** This is what watches your consumption folder for documents.
-    However, the consumer itself does not consume really consume your documents anymore.
-    It rather notifies a task processor that a new file is ready for consumption.
+    However, the consumer itself does not really consume your documents.
+    Now it notifies a task processor that a new file is ready for consumption.
     I suppose it should be named differently.
-    This also used to check your emails, but that's now gone elsewhere as well.
+    This was also used to check your emails, but that's now done elsewhere as well.
 
     Start the consumer with the management command ``document_consumer``:
 
     .. code:: shell-session
 
         $ cd /path/to/paperless/src/
-        $ pipenv run python3 manage.py document_consumer
+        $ python3 manage.py document_consumer
 
     .. _setup-task_processor:
 
 *   **The task processor:** Paperless relies on `Django Q <https://django-q.readthedocs.io/en/latest/>`_
-    for doing much of the heavy lifting. This is a task queue that accepts tasks from
-    multiple sources and processes tasks in parallel. It also comes with a scheduler that executes
+    for doing most of the heavy lifting. This is a task queue that accepts tasks from
+    multiple sources and processes these in parallel. It also comes with a scheduler that executes
     certain commands periodically.
 
     This task processor is responsible for:
 
     *   Consuming documents. When the consumer finds new documents, it notifies the task processor to
         start a consumption task.
-    *   Consuming emails. It periodically checks your configured accounts for new mails and
-        produces consumption tasks for any documents it finds.
     *   The task processor also performs the consumption of any documents you upload through
         the web interface.
-    *   Maintain the search index and the automatic matching algorithm. These are things that paperless
+    *   Consuming emails. It periodically checks your configured accounts for new emails and
+        notifies the task processor to consume the attachment of an email.
+    *   Maintaining the search index and the automatic matching algorithm. These are things that paperless
         needs to do from time to time in order to operate properly.
 
     This allows paperless to process multiple documents from your consumption folder in parallel! On
-    a modern multi core system, consumption with full ocr is blazing fast.
+    a modern multi core system, this makes the consumption process with full OCR blazingly fast.
 
-    The task processor comes with a built-in admin interface that you can use to see whenever any of the
+    The task processor comes with a built-in admin interface that you can use to check whenever any of the
     tasks fail and inspect the errors (i.e., wrong email credentials, errors during consuming a specific
     file, etc).
 
@@ -67,11 +67,11 @@ Paperless consists of the following components:
     .. code:: shell-session
 
         $ cd /path/to/paperless/src/
-        $ pipenv run python3 manage.py qcluster
+        $ python3 manage.py qcluster
 
 *   A `redis <https://redis.io/>`_ message broker: This is a really lightweight service that is responsible
-    for getting the tasks from the webserver and consumer to the task scheduler. These run in different
-    processes (maybe even on different machines!), and therefore, this is necessary.
+    for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different
+    process (maybe even on different machines!), and therefore, this is necessary.
 
 *   Optional: A database server. Paperless supports both PostgreSQL and SQLite for storing its data.
 
@@ -79,28 +79,58 @@ Paperless consists of the following components:
 Installation
 ############
 
-You can go multiple routes with setting up and running Paperless:
+You can go multiple routes to setup and run Paperless:
 
+* :ref:`Use the easy install docker script <setup-docker_script>`
 * :ref:`Pull the image from Docker Hub <setup-docker_hub>`
 * :ref:`Build the Docker image yourself <setup-docker_build>`
-* :ref:`Install Paperless directly on your system (bare metal) <setup-bare_metal>`
+* :ref:`Install Paperless directly on your system manually (bare metal) <setup-bare_metal>`
+* :ref:`Use ansible to install Paperless on your system automatically (bare metal) <setup-ansible>`
 
 The Docker routes are quick & easy. These are the recommended routes. This configures all the stuff
-from above automatically so that it just works and uses sensible defaults for all configuration options.
+from the above automatically so that it just works and uses sensible defaults for all configuration options.
+Here you find a cheat-sheet for docker beginners: `CLI Basics <https://sehn.tech/post/devops-with-docker/>`_
 
-The bare metal route is more complicated to setup but makes it easier
+The bare metal route is complicated to setup but makes it easier
 should you want to contribute some code back. You need to configure and
 run the above mentioned components yourself.
 
+The ansible route combines benefits of both options:
+the setup process is fully automated, reproducible and `idempotent <https://docs.ansible.com/ansible/latest/reference_appendices/glossary.html#Idempotency>`_,
+it includes the same sensible defaults, and it simultaneously provides the flexibility of a bare metal installation.
+
+.. _CLI Basics: https://sehn.tech/post/devops-with-docker/
+.. _idempotent: https://docs.ansible.com/ansible/latest/reference_appendices/glossary.html#Idempotency
+
+.. _setup-docker_script:
+
+Install Paperless from Docker Hub using the installation script
+===============================================================
+
+Paperless provides an interactive installation script. This script will ask you
+for a couple configuration options, download and create the necessary configuration files, pull the docker image, start paperless and create your user account. This script essentially
+performs all the steps described in :ref:`setup-docker_hub` automatically.
+
+1.  Make sure that docker and docker-compose are installed.
+2.  Download and run the installation script:
+
+    .. code:: shell-session
+
+        $ wget https://raw.githubusercontent.com/jonaswinkler/paperless-ng/master/install-paperless-ng.sh
+        $ chmod +x install-paperless-ng.sh
+        $ ./install-paperless-ng.sh
+
 .. _setup-docker_hub:
 
 Install Paperless from Docker Hub
 =================================
 
-1.  Go to the `/docker/compose directory on the project page <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/compose>`_
-    and download one of the ``docker-compose.*.yml`` files, depending on which database backend you
+1.  Login with your user and create a folder in your home-directory `mkdir -v ~/paperless-ng` to have a place for your configuration files and consumption directory.
+
+2.  Go to the `/docker/compose directory on the project page <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/compose>`_
+    and download one of the `docker-compose.*.yml` files, depending on which database backend you
     want to use. Rename this file to `docker-compose.yml`.
-    If you want to enable optional support for Office documents, download a file with ``-tika`` in its name.
+    If you want to enable optional support for Office documents, download a file with `-tika` in the file name.
     Download the ``docker-compose.env`` file and the ``.env`` file as well and store them
     in the same directory.
 
@@ -109,25 +139,26 @@ Install Paperless from Docker Hub
         For new installations, it is recommended to use PostgreSQL as the database
         backend.
 
-2.  Install `Docker`_ and `docker-compose`_.
+3.  Install `Docker`_ and `docker-compose`_.
 
     .. caution::
 
         If you want to use the included ``docker-compose.*.yml`` file, you
         need to have at least Docker version **17.09.0** and docker-compose
         version **1.17.0**.
+        To check do: `docker-compose -v` or `docker -v`
 
         See the `Docker installation guide`_ on how to install the current
         version of Docker for your operating system or Linux distribution of
-        choice. To get an up-to-date version of docker-compose, follow the
+        choice. To get the latest version of docker-compose, follow the
         `docker-compose installation guide`_ if your package repository doesn't
         include it.
 
         .. _Docker installation guide: https://docs.docker.com/engine/installation/
         .. _docker-compose installation guide: https://docs.docker.com/compose/install/
 
-3.  Modify ``docker-compose.yml`` to your preferences. You may want to change the path
-    to the consumption directory in this file. Find the line that specifies where
+4.  Modify ``docker-compose.yml`` to your preferences. You may want to change the path
+    to the consumption directory. Find the line that specifies where
     to mount the consumption directory:
 
     .. code::
@@ -143,31 +174,34 @@ Install Paperless from Docker Hub
     Don't change the part after the colon or paperless wont find your documents.
 
 
-4.  Modify ``docker-compose.env``, following the comments in the file. The
+5.  Modify ``docker-compose.env``, following the comments in the file. The
     most important change is to set ``USERMAP_UID`` and ``USERMAP_GID``
-    to the uid and gid of your user on the host system. This ensures that
+    to the uid and gid of your user on the host system. Use ``id -u`` and
+    ``id -g`` to get these.
+
+    This ensures that
     both the docker container and you on the host machine have write access
     to the consumption directory. If your UID and GID on the host system is
     1000 (the default for the first normal user on most systems), it will
-    work out of the box without any modifications.
+    work out of the box without any modifications. `id "username"` to check.
 
     .. note::
 
-        You can use any settings from the file ``paperless.conf.example`` in this file.
-        Have a look at :ref:`configuration` to see whats available.
+        You can copy any setting from the file ``paperless.conf.example`` and paste it here.
+        Have a look at :ref:`configuration` to see what's available.
 
     .. caution::
 
-        Certain file systems such as NFS network shares don't support file system
+        Some file systems such as NFS network shares don't support file system
         notifications with ``inotify``. When storing the consumption directory
-        on such a file system, paperless will be unable to pick up new files
+        on such a file system, paperless will not pick up new files
         with the default configuration. You will need to use ``PAPERLESS_CONSUMER_POLLING``,
         which will disable inotify. See :ref:`here <configuration-polling>`.
 
-5.  Run ``docker-compose up -d``. This will create and start the necessary
-    containers.
+6.  Run ``docker-compose pull``, followed by ``docker-compose up -d``.
+    This will pull the image, create and start the necessary containers.
 
-6.  To be able to login, you will need a super user. To create it, execute the
+7.  To be able to login, you will need a super user. To create it, execute the
     following command:
 
     .. code-block:: shell-session
@@ -175,19 +209,19 @@ Install Paperless from Docker Hub
         $ docker-compose run --rm webserver createsuperuser
 
     This will prompt you to set a username, an optional e-mail address and
-    finally a password.
+    finally a password (at least 8 characters).
 
-7.  The default ``docker-compose.yml`` exports the webserver on your local port
-    8000. If you haven't adapted this, you should now be able to visit your
-    Paperless instance at ``http://127.0.0.1:8000``. You can login with the
-    user and password you just created.
+8.  The default ``docker-compose.yml`` exports the webserver on your local port
+    8000. If you did not change this, you should now be able to visit your
+    Paperless instance at ``http://127.0.0.1:8000`` or your servers IP-Address:8000.
+    Use the login credentials you have created with the previous step.
 
 .. _Docker: https://www.docker.com/
 .. _docker-compose: https://docs.docker.com/compose/install/
 
 .. _setup-docker_build:
 
-Build the docker image yourself
+Build the Docker image yourself
 ===============================
 
 1.  Clone the entire repository of paperless:
@@ -208,7 +242,7 @@ Build the docker image yourself
 
         webserver:
             image: jonaswinkler/paperless-ng:latest
-    
+
     and replace it with a line that instructs docker-compose to build the image from the current working directory instead:
 
     .. code:: yaml
@@ -218,14 +252,14 @@ Build the docker image yourself
 
 4.  Run the ``compile-frontend.sh`` script. This requires ``node`` and ``npm >= v15``.
 
-5.  Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run
-    ``docker-compose up -d`` to start the containers, do
+5.  Follow steps 3 to 8 of :ref:`setup-docker_hub`. When asked to run
+    ``docker-compose pull`` to pull the image, do
 
     .. code:: shell-session
 
         $ docker-compose build
 
-    before that to build the image.
+    instead to build the image.
 
 .. _setup-bare_metal:
 
@@ -238,15 +272,14 @@ writing. Windows is not and will never be supported.
 
 1.  Install dependencies. Paperless requires the following packages.
 
-    *   ``python3`` 3.6, 3.7, 3.8 (3.9 is untested).
-    *   ``python3-pip``, optionally ``pipenv`` for package installation
+    *   ``python3`` 3.6, 3.7, 3.8, 3.9
+    *   ``python3-pip``
     *   ``python3-dev``
 
     *   ``fonts-liberation`` for generating thumbnails for plain text files
     *   ``imagemagick`` >= 6 for PDF conversion
     *   ``optipng`` for optimizing thumbnails
     *   ``gnupg`` for handling encrypted documents
-    *   ``libpoppler-cpp-dev`` for PDF to text conversion
     *   ``libpq-dev`` for PostgreSQL
     *   ``libmagic-dev`` for mime type detection
     *   ``mime-support`` for mime type detection
@@ -275,7 +308,7 @@ writing. Windows is not and will never be supported.
 2.  Install ``redis`` >= 5.0 and configure it to start automatically.
 
 3.  Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
-    to use PostgreSQL, SQLite is avialable as well.
+    to use PostgreSQL, SQLite is available as well.
 
 4.  Get the release archive from `<https://github.com/jonaswinkler/paperless-ng/releases>`_.
     If you clone the git repo as it is, you also have to compile the front end by yourself.
@@ -299,8 +332,14 @@ writing. Windows is not and will never be supported.
     *   Set ``PAPERLESS_OCR_LANGUAGE`` to the language most of your documents are written in.
     *   Set ``PAPERLESS_TIME_ZONE`` to your local time zone.
 
-6.  Setup permissions. Create a system users under which you wish to run paperless. Ensure that these directories exist
-    and that the user has write permissions to the following directories
+6.  Create a system user under which you wish to run paperless.
+
+    .. code:: shell-session
+
+        adduser paperless --system --home /opt/paperless --group
+
+7.  Ensure that these directories exist
+    and that the paperless user has write permissions to the following directories:
 
     *   ``/opt/paperless/media``
     *   ``/opt/paperless/data``
@@ -308,39 +347,47 @@ writing. Windows is not and will never be supported.
 
     Adjust as necessary if you configured different folders.
 
-7.  Install python requirements. Paperless comes with both Pipfiles for ``pipenv`` as well as with a ``requirements.txt``.
-    Both will install exactly the same requirements. It is up to you if you wish to use a virtual environment or not.
+8.  Install python requirements from the ``requirements.txt`` file.
+    It is up to you if you wish to use a virtual environment or not.
 
-8.  Go to ``/opt/paperless/src``, and execute the following commands:
+    .. code:: shell-session
+
+        sudo -Hu paperless pip3 install -r requirements.txt
+
+    This will install all python dependencies in the home directory of
+    the new paperless user.
+
+9.  Go to ``/opt/paperless/src``, and execute the following commands:
 
     .. code:: bash
 
         # This creates the database schema.
-        python3 manage.py migrate
+        sudo -Hu paperless python3 manage.py migrate
 
         # This creates your first paperless user
-        python3 manage.py createsuperuser
+        sudo -Hu paperless python3 manage.py createsuperuser
 
-9.  Optional: Test that paperless is working by executing
+10. Optional: Test that paperless is working by executing
 
       .. code:: bash
 
         # This collects static files from paperless and django.
-        python3 manage.py runserver
+        sudo -Hu paperless python3 manage.py runserver
 
     and pointing your browser to http://localhost:8000/.
 
     .. warning::
 
         This is a development server which should not be used in
-        production.
+        production. It is not audited for security and performance
+        is inferior to production ready web servers.
 
     .. hint::
 
         This will not start the consumer. Paperless does this in a
         separate process.
 
-10. Setup systemd services to run paperless automatically. You may
+11. Setup systemd services to run paperless automatically. You may
     use the service definition files included in the ``scripts`` folder
     as a starting point.
 
@@ -348,17 +395,15 @@ writing. Windows is not and will never be supported.
     ``consumer`` script to watch the input folder, and the ``scheduler``
     script to run tasks such as email checking and document consumption.
 
+    You may need to adjust the path to the ``gunicorn`` executable. This
+    will be installed as part of the python dependencies, and is either located
+    in the ``bin`` folder of your virtual environment, or in ``~/.local/bin/`` if
+    no virtual environment is used.
+
     These services rely on redis and optionally the database server, but
     don't need to be started in any particular order. The example files
     depend on redis being started. If you use a database server, you should
-    add additinal dependencies.
-
-    .. hint::
-
-        You may optionally set up your preferred web server to serve
-        paperless as a wsgi application directly instead of running the
-        ``webserver`` service. The module containing the wsgi application
-        is named ``paperless.wsgi``.
+    add additional dependencies.
 
     .. caution::
 
@@ -367,10 +412,13 @@ writing. Windows is not and will never be supported.
         however, the documentation of GUnicorn states that you should
         use a proxy server in front of gunicorn instead.
 
-11. Optional: Install a samba server and make the consumption folder
+        For instructions on how to use nginx for that,
+        :ref:`see the instructions below <setup-nginx>`.
+
+12. Optional: Install a samba server and make the consumption folder
     available as a network share.
 
-12. Configure ImageMagick to allow processing of PDF documents. Most distributions have
+13. Configure ImageMagick to allow processing of PDF documents. Most distributions have
     this disabled by default, since PDF documents can contain malware. If
     you don't do this, paperless will fall back to ghostscript for certain steps
     such as thumbnail generation.
@@ -387,17 +435,136 @@ writing. Windows is not and will never be supported.
 
         <policy domain="coder" rights="read|write" pattern="PDF" />
 
-13. Optional: Install the `jbig2enc <https://ocrmypdf.readthedocs.io/en/latest/jbig2.html>`_
+14. Optional: Install the `jbig2enc <https://ocrmypdf.readthedocs.io/en/latest/jbig2.html>`_
     encoder. This will reduce the size of generated PDF documents. You'll most likely need
     to compile this by yourself, because this software has been patented until around 2017 and
     binary packages are not available for most distributions.
 
+.. _setup-ansible:
+
+Install Paperless using ansible
+===============================
+
+.. note::
+
+    This role currently only supports Debian 10 Buster and Ubuntu 20.04 Focal or later as target hosts.
+
+1.  Install ansible 2.7+ on the management node.
+    This may be the target host paperless-ng is being installed on or any remote host which can access the target host.
+    For further details, check the ansible `inventory <https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html>`_ documentation.
+
+    On Debian and Ubuntu, the official repositories should provide a suitable version:
+
+    .. code:: bash
+
+        apt install ansible
+        ansible --version
+
+    Alternatively, you can install the most recent ansible release using PyPI:
+
+    .. code:: bash
+
+        python3 -m pip install ansible
+        ansible --version
+
+    Make sure your taget hosts are accessible:
+
+    .. code:: sh
+
+        ansible -m ping YourAnsibleTargetHostGoesHere
+
+2.  Clone the repository of paperless-ng:
+
+    .. code:: sh
+
+        git clone https://github.com/jonaswinkler/paperless-ng
+
+    Checkout the latest release tag:
+
+    .. code:: sh
+
+        cd paperless-ng
+        git checkout ng-1.0.0
+
+3.  Create an ansible ``playbook.yml`` in the paperless-ng root directory:
+
+    .. code:: yaml
+
+        - hosts: YourAnsibleTargetHostGoesHere
+          become: yes
+          vars_files:
+            - ansible/vars.yml
+          roles:
+            - ansible
+
+    Optional: If you also want to use PostgreSQL on the target system, install and add (for example) the `geerlingguy.postgresql <https://github.com/geerlingguy/ansible-role-postgresql>`_ role:
+
+    .. code:: sh
+
+        ansible-galaxy install geerlingguy.postgresql
+
+    .. code:: yaml
+
+        - hosts: YourAnsibleTargetHostGoesHere
+          become: yes
+          vars_files:
+            - ansible/vars.yml
+          roles:
+            - geerlingguy.postgresql
+            - ansible
+
+    Optional: If you also want to use a reverse proxy on the target system, install and add (for example) the `geerlingguy.nginx <https://github.com/geerlingguy/ansible-role-nginx>`_ role:
+
+    .. code:: sh
+
+        ansible-galaxy install geerlingguy.nginx
+
+    .. code:: yaml
+
+        - hosts: YourAnsibleTargetHostGoesHere
+          become: yes
+          vars_files:
+            - ansible/vars.yml
+          roles:
+            - geerlingguy.postgresql
+            - ansible
+            - geerlingguy.nginx
+
+4.  Create ``ansible/vars.yml`` to configure your ansible deployment:
+
+    .. code:: yaml
+
+        paperlessng_secret_key: PleaseGenerateAStrongKeyForThis
+
+        paperlessng_superuser_name: YourUserName
+        paperlessng_superuser_email: name@domain.tld
+        paperlessng_superuser_password: YourDesiredPasswordUsedForFirstLogin
+
+        paperlessng_ocr_languages:
+            - eng
+            - deu
+
+    For all of the available options, please check ``ansible/README.md`` and :ref:`configuration`.
+
+    Optional configurations for the above-mentioned PostgreSQL and nginx roles would also go here.
+
+5. Run the ansible playbook from the management node:
+
+    .. code:: sh
+
+        ansible-playbook playbook.yml
+
+    When this step completes successfully, paperless-ng will be available on the target host at ``http://127.0.0.1:8000`` (or the address you configured).
+
 Migration to paperless-ng
 #########################
 
 At its core, paperless-ng is still paperless and fully compatible. However, some
 things have changed under the hood, so you need to adapt your setup depending on
-how you installed paperless. The important things to keep in mind are as follows.
+how you installed paperless.
+
+This setup describes how to update an existing paperless Docker installation.
+The important things to keep in mind are as follows:
 
 * Read the :ref:`changelog <paperless_changelog>` and take note of breaking changes.
 * You should decide if you want to stick with SQLite or want to migrate your database
@@ -425,18 +592,25 @@ Migration to paperless-ng is then performed in a few simple steps:
     paperless.
 
 3.  Download the latest release of paperless-ng. You can either go with the
-    docker-compose files from `here <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/compose>`_
+    docker-compose files from `here <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/compose>`__
     or clone the repository to build the image yourself (see :ref:`above <setup-docker_build>`).
     You can either replace your current paperless folder or put paperless-ng
     in a different location.
 
     .. caution::
 
-        Paperless includes a ``.env`` file. This will set the
-        project name for docker compose to ``paperless`` so that paperless-ng will
-        automatically reuse your existing paperless volumes. When you start it, it
-        will migrate your existing data. After that, your old paperless installation
-        will be incompatible with the migrated volumes.
+        Paperless-ng includes a ``.env`` file. This will set the
+        project name for docker compose to ``paperless``, which will also define the name
+        of the volumes by paperless-ng. However, if you experience that paperless-ng
+        is not using your old paperless volumes, verify the names of your volumes with
+
+        .. code:: shell-session
+
+            $ docker volume ls | grep _data
+
+        and adjust the project name in the ``.env`` file so that it matches the name
+        of the volumes before the ``_data`` part.
+
 
 4.  Download the ``docker-compose.sqlite.yml`` file to ``docker-compose.yml``.
     If you want to switch to PostgreSQL, do that after you migrated your existing
@@ -517,14 +691,12 @@ management commands as below.
 
         This will launch the container and initialize the PostgreSQL database.
 
-    b)  Without docker, open a shell in your virtual environment, switch to
+    b)  Without docker, remember to activate any virtual environment, switch to
         the ``src`` directory and create the database schema:
 
         .. code:: shell-session
 
-            $ cd /path/to/paperless
-            $ pipenv shell
-            $ cd src
+            $ cd /path/to/paperless/src
             $ python3 manage.py migrate
 
         This will not copy any data yet.
@@ -541,7 +713,7 @@ management commands as below.
 
         $ python3 manage.py loaddata data.json
 
-6.  Exit the shell.
+6.  If operating inside Docker, you may exit the shell now.
 
     .. code:: shell-session
 
@@ -590,7 +762,8 @@ configuring some options in paperless can help improve performance immensely:
 
 *   Stick with SQLite to save some resources.
 *   Consider setting ``PAPERLESS_OCR_PAGES`` to 1, so that paperless will only OCR
-    the first page of your documents.
+    the first page of your documents. In most cases, this page contains enough
+    information to be able to find it.
 *   ``PAPERLESS_TASK_WORKERS`` and ``PAPERLESS_THREADS_PER_WORKER`` are configured
     to use all cores. The Raspberry Pi models 3 and up have 4 cores, meaning that
     paperless will use 2 workers and 2 threads per worker. This may result in
@@ -601,8 +774,13 @@ configuring some options in paperless can help improve performance immensely:
     your documents before feeding them into paperless. Some scanners are able to
     do this! You might want to even specify ``skip_noarchive`` to skip archive
     file generation for already ocr'ed documents entirely.
+*   If you want to perform OCR on the the device, consider using ``PAPERLESS_OCR_CLEAN=none``.
+    This will speed up OCR times and use less memory at the expense of slightly worse
+    OCR results.
 *   Set ``PAPERLESS_OPTIMIZE_THUMBNAILS`` to 'false' if you want faster consumption
     times. Thumbnails will be about 20% larger.
+*   If using docker, consider setting ``PAPERLESS_WEBSERVER_WORKERS`` to
+    1. This will save some memory.
 
 For details, refer to :ref:`configuration`.
 
@@ -619,3 +797,46 @@ For details, refer to :ref:`configuration`.
     well as on any other device.
 
 .. _redis: https://redis.io/
+
+
+.. _setup-nginx:
+
+Using nginx as a reverse proxy
+##############################
+
+If you want to expose paperless to the internet, you should hide it behind a
+reverse proxy with SSL enabled.
+
+In addition to the usual configuration for SSL,
+the following configuration is required for paperless to operate:
+
+.. code:: nginx
+
+    http {
+
+        # Adjust as required. This is the maximum size for file uploads.
+        # The default value 1M might be a little too small.
+        client_max_body_size 10M;
+
+        server {
+
+            location / {
+
+                # Adjust host and port as required.
+                proxy_pass http://localhost:8000/;
+
+                # These configuration options are required for WebSockets to work.
+                proxy_http_version 1.1;
+                proxy_set_header Upgrade $http_upgrade;
+                proxy_set_header Connection "upgrade";
+
+                proxy_redirect off;
+                proxy_set_header Host $host;
+                proxy_set_header X-Real-IP $remote_addr;
+                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                proxy_set_header X-Forwarded-Host $server_name;
+            }
+        }
+    }
+
+Also read `this <https://channels.readthedocs.io/en/stable/deploying.html#nginx-supervisor-ubuntu>`__, towards the end of the section.

docs/troubleshooting.rst

@@ -30,13 +30,22 @@ Consumer fails to pickup any new files
 ######################################
 
 If you notice that the consumer will only pickup files in the consumption
-directory at startup, but won't find any other files added later, check out
-the configuration file and enable filesystem polling with the setting
-``PAPERLESS_CONSUMER_POLLING``.
+directory at startup, but won't find any other files added later, you will need to
+enable filesystem polling with the configuration option
+``PAPERLESS_CONSUMER_POLLING``, see :ref:`here <configuration-polling>`.
 
 This will disable listening to filesystem changes with inotify and paperless will
 manually check the consumption directory for changes instead.
 
+
+Paperless always redirects to /admin
+####################################
+
+You probably had the old paperless installed at some point. Paperless installed
+a permanent redirect to /admin in your browser, and you need to clear your
+browsing data / cache to fix that.
+
+
 Operation not permitted
 #######################
 
@@ -51,7 +60,8 @@ required so that the user running paperless inside docker has write permissions
 to these folders. This happens when pointing these directories to NFS shares,
 for example.
 
-Ensure that `chown` is possible on these directories.
+Ensure that ``chown`` is possible on these directories.
+
 
 Classifier error: No training data available
 ############################################
@@ -64,6 +74,50 @@ This may have two reasons:
     with Inbox tags. Verify that there are documents in your archive without inbox tags.
     The algorithm will only learn from documents not in your inbox.
 
+
+UserWarning in sklearn on every single document
+###############################################
+
+You may encounter warnings like this:
+
+.. code::
+    
+    /usr/local/lib/python3.7/site-packages/sklearn/base.py:315:
+    UserWarning: Trying to unpickle estimator CountVectorizer from version 0.23.2 when using version 0.24.0.
+    This might lead to breaking code or invalid results. Use at your own risk.
+
+This happens when certain dependencies of paperless that are responsible for the auto matching algorithm are
+updated. After updating these, your current training data *might* not be compatible anymore. This can be ignored
+in most cases. This warning will disappear automatically when paperless updates the training data.
+
+If you want to get rid of the warning or actually experience issues with automatic matching, delete
+the file ``classification_model.pickle`` in the data directory and let paperless recreate it.
+
+
+504 Server Error: Gateway Timeout when adding Office documents
+##############################################################
+
+You may experience these errors when using the optional TIKA integration:
+
+.. code::
+
+    requests.exceptions.HTTPError: 504 Server Error: Gateway Timeout for url: http://gotenberg:3000/convert/office
+
+Gotenberg is a server that converts Office documents into PDF documents and has a default timeout of 10 seconds.
+When conversion takes longer, Gotenberg raises this error.
+
+You can increase the timeout by configuring an environment variable for gotenberg (see also `here <https://thecodingmachine.github.io/gotenberg/#environment_variables.default_wait_timeout>`__).
+If using docker-compose, this is achieved by the following configuration change in the ``docker-compose.yml`` file:
+
+.. code:: yaml
+
+    gotenberg:
+        image: thecodingmachine/gotenberg
+        restart: unless-stopped
+        environment:
+            DISABLE_GOOGLE_CHROME: 1
+            DEFAULT_WAIT_TIMEOUT: 30
+
 Permission denied errors in the consumption directory
 #####################################################
 
@@ -78,3 +132,79 @@ Ensure that ``USERMAP_UID`` and ``USERMAP_GID`` are set to the user id and group
 different from ``1000``. See :ref:`setup-docker_hub`.
 
 Also ensure that you are able to read and write to the consumption directory on the host.
+
+
+OSError: [Errno 19] No such device when consuming files
+#######################################################
+
+If you experience errors such as:
+
+.. code:: shell-session
+
+    File "/usr/local/lib/python3.7/site-packages/whoosh/codec/base.py", line 570, in open_compound_file
+    return CompoundStorage(dbfile, use_mmap=storage.supports_mmap)
+    File "/usr/local/lib/python3.7/site-packages/whoosh/filedb/compound.py", line 75, in __init__
+    self._source = mmap.mmap(fileno, 0, access=mmap.ACCESS_READ)
+    OSError: [Errno 19] No such device
+
+    During handling of the above exception, another exception occurred:
+
+    Traceback (most recent call last):
+    File "/usr/local/lib/python3.7/site-packages/django_q/cluster.py", line 436, in worker
+    res = f(*task["args"], **task["kwargs"])
+    File "/usr/src/paperless/src/documents/tasks.py", line 73, in consume_file
+    override_tag_ids=override_tag_ids)
+    File "/usr/src/paperless/src/documents/consumer.py", line 271, in try_consume_file
+    raise ConsumerError(e)
+
+Paperless uses a search index to provide better and faster full text searching. This search index is stored inside
+the ``data`` folder. The search index uses memory-mapped files (mmap). The above error indicates that paperless
+was unable to create and open these files.
+
+This happens when you're trying to store the data directory on certain file systems (mostly network shares)
+that don't support memory-mapped files.
+
+
+Web-UI stuck at "Loading..."
+############################
+
+This might have multiple reasons.
+
+
+1.  If you built the docker image yourself or deployed using the bare metal route,
+    make sure that there are files in ``<paperless-root>/static/frontend/<lang-code>/``.
+    If there are no files, make sure that you executed ``collectstatic`` successfully, either
+    manually or as part of the docker image build.
+
+    If the front end is still missing, make sure that the front end is compiled (files present in
+    ``src/documents/static/frontend``). If it is not, you need to compile the front end yourself
+    or download the release archive instead of cloning the repository.
+
+2.  Check the output of the web server. You might see errors like this:
+
+
+    .. code::
+
+        [2021-01-25 10:08:04 +0000] [40] [ERROR] Socket error processing request.
+        Traceback (most recent call last):
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 134, in handle
+            self.handle_request(listener, req, client, addr)
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 190, in handle_request
+            util.reraise(*sys.exc_info())
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/util.py", line 625, in reraise
+            raise value
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 178, in handle_request
+            resp.write_file(respiter)
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/http/wsgi.py", line 396, in write_file
+            if not self.sendfile(respiter):
+        File "/usr/local/lib/python3.7/site-packages/gunicorn/http/wsgi.py", line 386, in sendfile
+            sent += os.sendfile(sockno, fileno, offset + sent, count)
+        OSError: [Errno 22] Invalid argument
+    
+    To fix this issue, add
+
+    .. code::
+
+        SENDFILE=0
+    
+    to your `docker-compose.env` file.

gunicorn.conf.py

@@ -1,21 +1,9 @@
+import os
+
 bind = '0.0.0.0:8000'
-backlog = 2048
-workers = 3
-worker_class = 'sync'
-worker_connections = 1000
-timeout = 20
-keepalive = 2
-spew = False
-daemon = False
-pidfile = None
-umask = 0
-user = None
-group = None
-tmp_upload_dir = None
-loglevel = 'info'
-errorlog = '-'
-accesslog = '-'
-proc_name = None
+workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 2))
+worker_class = 'uvicorn.workers.UvicornWorker'
+timeout = 120
 
 def pre_fork(server, worker):
     pass

install-paperless-ng.sh

@@ -0,0 +1,295 @@
+#!/bin/bash
+
+set -e
+
+ask() {
+	while true ; do
+		if [[ -z $3 ]] ; then
+			read -p "$1 [$2]: " result
+		else
+			read -p "$1 ($3) [$2]: " result
+		fi
+		if [[ -z $result ]]; then
+			ask_result=$2
+			return
+		fi
+		array=$3
+		if [[ -z $3 || " ${array[@]} " =~ " ${result} " ]]; then
+			ask_result=$result
+			return
+		else
+			echo "Invalid option: $result"
+		fi
+	done
+}
+
+ask_docker_folder() {
+	while true ; do
+
+		read -p "$1 [$2]: " result
+
+		if [[ -z $result ]]; then
+			ask_result=$2
+			return
+		fi
+
+		if [[ $result == /* || $result == ./* ]]; then
+			ask_result=$result
+			return
+		else
+			echo "Invalid folder: $result"
+		fi
+
+
+	done
+}
+
+if [[ $(id -u) == "0" ]] ; then
+	echo "Do not run this script as root."
+	exit 1
+fi
+
+if [[ -z $(which wget) ]] ; then
+	echo "wget executable not found. Is wget installed?"
+	exit 1
+fi
+
+if [[ -z $(which docker) ]] ; then
+	echo "docker executable not found. Is docker installed?"
+	exit 1
+fi
+
+if [[ -z $(which docker-compose) ]] ; then
+	echo "docker-compose executable not found. Is docker-compose installed?"
+	exit 1
+fi
+
+echo ""
+echo "############################################"
+echo "###   Paperless-ng docker installation   ###"
+echo "############################################"
+echo ""
+echo "This script will download, configure and start paperless-ng."
+
+echo ""
+echo "1. Folder configuration"
+echo "======================="
+echo ""
+echo "The target folder is used to store the configuration files of "
+echo "paperless. You can move this folder around after installing paperless."
+echo "You will need this folder whenever you want to start, stop, update or "
+echo "maintain your paperless instance."
+echo ""
+
+ask "Target folder" "$(pwd)/paperless-ng"
+TARGET_FOLDER=$ask_result
+
+echo ""
+echo "The consume folder is where paperles will search for new documents."
+echo "Point this to a folder where your scanner is able to put your scanned"
+echo "documents."
+echo ""
+echo "CAUTION: You must specify an absolute path starting with / or a relative "
+echo "path starting with ./ here. Examples:"
+echo "  /mnt/consume"
+echo "  ./consume"
+echo ""
+
+ask_docker_folder "Consume folder" "$TARGET_FOLDER/consume"
+CONSUME_FOLDER=$ask_result
+
+echo ""
+echo "The media folder is where paperless stores your documents."
+echo "Leave empty and docker will manage this folder for you."
+echo "Docker usually stores managed folders in /var/lib/docker/volumes."
+echo ""
+echo "CAUTION: If specified, you must specify an absolute path starting with /"
+echo "or a relative path starting with ./ here."
+echo ""
+
+ask_docker_folder "Media folder" ""
+MEDIA_FOLDER=$ask_result
+
+echo ""
+echo "The data folder is where paperless stores other data, such as your"
+echo "SQLite database (if used), the search index and other data."
+echo "As with the media folder, leave empty to have this managed by docker."
+echo ""
+echo "CAUTION: If specified, you must specify an absolute path starting with /"
+echo "or a relative path starting with ./ here."
+echo ""
+
+ask_docker_folder "Data folder" ""
+DATA_FOLDER=$ask_result
+
+echo ""
+echo "2. Application configuration"
+echo "============================"
+
+echo ""
+echo "The port on which the paperless webserver will listen for incoming"
+echo "connections."
+echo ""
+
+ask "Port" "8000"
+PORT=$ask_result
+
+echo ""
+echo "Database backend: PostgreSQL and SQLite are available. Use PostgreSQL"
+echo "if unsure. If you're running on a low-power device such as Raspberry"
+echo "Pi, use SQLite to save resources."
+echo ""
+
+ask "Database backend" "postgres" "postgres sqlite"
+DATABASE_BACKEND=$ask_result
+
+echo ""
+echo "Paperless is able to use Apache Tika to support Office documents such as"
+echo "Word, Excel, Powerpoint, and Libreoffice equivalents. This feature"
+echo "requires more resources due to the required services."
+echo ""
+
+ask "Enable Apache Tika?" "no" "yes no"
+TIKA_ENABLED=$ask_result
+
+echo ""
+echo "Specify the default language that most of your documents are written in."
+echo "Use ISO 639-2, (T) variant language codes: "
+echo "https://www.loc.gov/standards/iso639-2/php/code_list.php"
+echo "Common values: eng (English) deu (German) nld (Dutch) fra (French)"
+echo ""
+
+ask "OCR language" "eng"
+OCR_LANGUAGE=$ask_result
+
+echo ""
+echo "Specify the user id and group id you wish to run paperless as."
+echo "Paperless will also change ownership on the data, media and consume"
+echo "folder to the specified values, so it's a good idea to supply the user id"
+echo "and group id of your unix user account."
+echo "If unsure, leave default."
+echo ""
+
+ask "User ID" "$(id -u)"
+USERMAP_UID=$ask_result
+
+ask "Group ID" "$(id -g)"
+USERMAP_GID=$ask_result
+
+echo ""
+echo "3. Login credentials"
+echo "===================="
+echo ""
+echo "Specify initial login credentials. You can change these later."
+echo "A mail address is required, however it is not used in paperless. You don't"
+echo "need to provide an actual mail address."
+echo ""
+
+ask "Paperless username" "$(whoami)"
+USERNAME=$ask_result
+
+while true; do
+	read -sp "Paperless password: " PASSWORD
+	echo ""
+
+	if [[ -z $PASSWORD ]] ; then
+		echo "Password cannot be empty."
+		continue
+	fi
+
+	read -sp "Paperless password (again): " PASSWORD_REPEAT
+	echo ""
+
+	if [[ ! "$PASSWORD" == "$PASSWORD_REPEAT" ]] ; then
+		echo "Passwords did not match"
+	else
+		break
+	fi
+done
+
+ask "Email" "$USERNAME@localhost"
+EMAIL=$ask_result
+
+echo ""
+echo "Summary"
+echo "======="
+echo ""
+
+echo "Target folder: $TARGET_FOLDER"
+echo "Consume folder: $CONSUME_FOLDER"
+if [[ -z $MEDIA_FOLDER ]] ; then
+	echo "Media folder: Managed by docker"
+else
+	echo "Media folder: $MEDIA_FOLDER"
+fi
+if [[ -z $DATA_FOLDER ]] ; then
+	echo "Data folder: Managed by docker"
+else
+	echo "Data folder: $DATA_FOLDER"
+fi
+echo ""
+echo "Port: $PORT"
+echo "Database: $DATABASE_BACKEND"
+echo "Tika enabled: $TIKA_ENABLED"
+echo "OCR language: $OCR_LANGUAGE"
+echo "User id: $USERMAP_UID"
+echo "Group id: $USERMAP_GID"
+echo ""
+echo "Paperless username: $USERNAME"
+echo "Paperless email: $EMAIL"
+
+echo ""
+read -p "Press any key to install."
+
+echo ""
+echo "Installing paperless..."
+echo ""
+
+mkdir -p "$TARGET_FOLDER"
+
+cd "$TARGET_FOLDER"
+
+DOCKER_COMPOSE_VERSION=$DATABASE_BACKEND
+
+if [[ $TIKA_ENABLED == "yes" ]] ; then
+	DOCKER_COMPOSE_VERSION="$DOCKER_COMPOSE_VERSION-tika"
+fi
+
+wget "https://raw.githubusercontent.com/jonaswinkler/paperless-ng/master/docker/compose/docker-compose.$DOCKER_COMPOSE_VERSION.yml" -O docker-compose.yml
+wget "https://raw.githubusercontent.com/jonaswinkler/paperless-ng/master/docker/compose/.env" -O .env
+
+SECRET_KEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1)
+
+DEFAULT_LANGUAGES="deu eng fra ita spa"
+
+{
+	if [[ ! $USERMAP_UID == "1000" ]] ; then
+		echo "USERMAP_UID=$USERMAP_UID"
+	fi
+	if [[ ! $USERMAP_GID == "1000" ]] ; then
+		echo "USERMAP_GID=$USERMAP_GID"
+	fi
+	echo "PAPERLESS_OCR_LANGUAGE=$OCR_LANGUAGE"
+	echo "PAPERLESS_SECRET_KEY=$SECRET_KEY"
+	if [[ ! " ${DEFAULT_LANGUAGES[@]} " =~ " ${OCR_LANGUAGE} " ]] ; then
+		echo "PAPERLESS_OCR_LANGUAGES=$OCR_LANGUAGE"
+	fi
+} > docker-compose.env
+
+sed -i "s/- 8000:8000/- $PORT:8000/g" docker-compose.yml
+
+sed -i "s#- \./consume:/usr/src/paperless/consume#- $CONSUME_FOLDER:/usr/src/paperless/consume#g" docker-compose.yml
+
+if [[ -n $MEDIA_FOLDER ]] ; then
+	sed -i "s#- media:/usr/src/paperless/media#- $MEDIA_FOLDER:/usr/src/paperless/media#g" docker-compose.yml
+fi
+
+if [[ -n $DATA_FOLDER ]] ; then
+	sed -i "s#- data:/usr/src/paperless/data#- $DATA_FOLDER:/usr/src/paperless/data#g" docker-compose.yml
+fi
+
+docker-compose pull
+
+docker-compose run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
+
+docker-compose up -d

paperless.conf.example

@@ -27,7 +27,7 @@
 
 #PAPERLESS_SECRET_KEY=change-me
 #PAPERLESS_ALLOWED_HOSTS=example.com,www.example.com
-#PAPERLESS_CORS_ALLOWED_HOSTS=localhost:8080,example.com,localhost:8000
+#PAPERLESS_CORS_ALLOWED_HOSTS=http://example.com,http://localhost:8000
 #PAPERLESS_FORCE_SCRIPT_NAME=
 #PAPERLESS_STATIC_URL=/static/
 #PAPERLESS_AUTO_LOGIN_USERNAME=
@@ -41,6 +41,10 @@
 #PAPERLESS_OCR_OUTPUT_TYPE=pdfa
 #PAPERLESS_OCR_PAGES=1
 #PAPERLESS_OCR_IMAGE_DPI=300
+#PAPERLESS_OCR_CLEAN=clean
+#PAPERLESS_OCR_DESKEW=true
+#PAPERLESS_OCR_ROTATE_PAGES=true
+#PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0
 #PAPERLESS_OCR_USER_ARGS={}
 #PAPERLESS_CONVERT_MEMORY_LIMIT=0
 #PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless

requirements.txt

@@ -7,68 +7,95 @@
 
 -i https://pypi.python.org/simple
 --extra-index-url https://www.piwheels.org/simple
+aioredis==1.3.1
 arrow==0.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 asgiref==3.3.1; python_version >= '3.5'
+async-timeout==3.0.1; python_full_version >= '3.5.3'
+attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+autobahn==21.2.1; python_version >= '3.6'
+automat==20.2.0
 blessed==1.17.12
 certifi==2020.12.5
-cffi==1.14.4
+cffi==1.14.5
+channels-redis==3.2.0
+channels==3.0.3
 chardet==4.0.0; python_version >= '3.1'
+click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+concurrent-log-handler==0.9.19
+constantly==15.1.0
+cryptography==3.3.2
+daphne==3.0.1; python_version >= '3.6'
 dateparser==0.7.6
-django-cors-headers==3.6.0
-django-extensions==3.1.0
+django-cors-headers==3.7.0
+django-extensions==3.1.1
 django-filter==2.4.0
 django-picklefield==3.0.1; python_version >= '3'
 django-q==1.3.4
-django==3.1.5
+django==3.1.7
 djangorestframework==3.12.2
 filelock==3.0.12
-fuzzywuzzy==0.18.0
+fuzzywuzzy[speedup]==0.18.0
 gunicorn==20.0.4
+h11==0.12.0; python_version >= '3.6'
+hiredis==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+httptools==0.1.1
 humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+hyperlink==21.0.0
 idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-imap-tools==0.34.0
+imap-tools==0.37.0
 img2pdf==0.4.0
-importlib-metadata==3.4.0; python_version < '3.8'
+incremental==17.5.0
 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 inotifyrecursive==0.3.5
-joblib==1.0.0; python_version >= '3.6'
+joblib==1.0.1; python_version >= '3.6'
 langdetect==1.0.8
 lxml==4.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-numpy==1.19.5; python_version >= '3.6'
-ocrmypdf==11.4.5
+msgpack==1.0.2
+numpy==1.19.5
+ocrmypdf==11.6.2
 pathvalidate==2.3.2
-pdfminer.six==20201018; python_version >= '3.4'
-pdftotext==2.1.5
-pikepdf==2.2.5
+pdfminer.six==20201018
+pikepdf==2.5.2
 pillow==8.1.0
 pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+portalocker==2.2.1; python_version >= '3'
 psycopg2-binary==2.8.6
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
 pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pyhamcrest==2.0.2; python_version >= '3.5'
+pyopenssl==20.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 python-dateutil==2.8.1
 python-dotenv==0.15.0
 python-gnupg==0.4.6
-python-levenshtein==0.12.0
-python-magic==0.4.18
-pytz==2020.5
+python-levenshtein==0.12.2
+python-magic==0.4.22
+pytz==2021.1
+pyyaml==5.4.1
 redis==3.5.3
 regex==2020.11.13
 reportlab==3.5.59
 requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 scikit-learn==0.24.0
-scipy==1.5.4; python_version >= '3.6'
+scipy==1.5.4
+service-identity==18.1.0
 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sortedcontainers==2.3.0
 sqlparse==0.4.1; python_version >= '3.5'
 threadpoolctl==2.1.0; python_version >= '3.5'
 tika==1.24
-tqdm==4.56.0
-typing-extensions==3.7.4.3; python_version < '3.8'
+tqdm==4.57.0
+twisted[tls]==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+txaio==21.2.1; python_version >= '3.6'
 tzlocal==2.1
-urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+urllib3==1.26.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+uvicorn[standard]==0.13.4
+uvloop==0.14.0
 watchdog==1.0.2
+watchgod==0.7
 wcwidth==0.2.5
+websockets==8.1
 whitenoise==5.2.0
 whoosh==2.7.4
-zipp==3.4.0; python_version >= '3.6'
+zope.interface==5.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

resources/logo.txt

@@ -0,0 +1,16 @@
+                         9w
+                       {@@N
+                     Q@@@@H
+                  G@@@@@@@\
+               SilN@@@@@@@
+            *Q  *@@@@@@@@S        /= = = = = = = = = = = = = = = = = =\
+          *@   B@@@@@@@@N        ||                                   ||
+ N       R$  A@@@@@@@@@@         ||           PAPERLESS-NG            ||
+x@@     $U  B@@@@@@@@@R          ||                                   ||
+N@@N^   @  N@@@@@@@@@*            \= = = = = = = = = = = = = = = = = =/
+|@@@u   @ E@@@@@@@@l
+ Q@@@ \ Px@@@@@@P
+  1@@S`  @@@o'
+    z$  ;
+       v
+      /

scripts/paperless-webserver.service

@@ -8,7 +8,7 @@ Requires=redis.service
 User=paperless
 Group=paperless
 WorkingDirectory=/opt/paperless/src
-ExecStart=/opt/paperless/.local/bin/gunicorn paperless.wsgi -w 2 -b 0.0.0.0:8000
+ExecStart=/opt/paperless/.local/bin/gunicorn -c /opt/paperless/gunicorn.conf.py paperless.asgi:application
 
 [Install]
 WantedBy=multi-user.target

src-ui/angular.json

@@ -18,7 +18,8 @@
 				"locales": {
 					"de": "src/locale/messages.de.xlf",
 					"nl-NL": "src/locale/messages.nl_NL.xlf",
-					"fr": "src/locale/messages.fr.xlf"
+					"fr": "src/locale/messages.fr.xlf",
+					"en-GB": "src/locale/messages.en_GB.xlf"
 				}
 			},
 			"architect": {

src-ui/package-lock.json

@@ -5545,6 +5545,11 @@
         "schema-utils": "^2.6.5"
       }
     },
+    "file-saver": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.5.tgz",
+      "integrity": "sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA=="
+    },
     "file-uri-to-path": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",

src-ui/package.json

@@ -23,6 +23,7 @@
     "@ng-bootstrap/ng-bootstrap": "^8.0.0",
     "@ng-select/ng-select": "^5.0.9",
     "bootstrap": "^4.5.0",
+    "file-saver": "^2.0.5",
     "ng-bootstrap": "^1.6.3",
     "ng2-pdf-viewer": "^6.3.2",
     "ngx-cookie-service": "^10.1.1",

src-ui/src/app/app.component.ts

@@ -1,17 +1,70 @@
-import { Component } from '@angular/core';
-import { SettingsService } from './services/settings.service';
+import { SettingsService, SETTINGS_KEYS } from './services/settings.service';
+import { Component, OnDestroy, OnInit } from '@angular/core';
+import { Router } from '@angular/router';
+import { Subscription } from 'rxjs';
+import { ConsumerStatusService } from './services/consumer-status.service';
+import { ToastService } from './services/toast.service';
 
 @Component({
   selector: 'app-root',
   templateUrl: './app.component.html',
   styleUrls: ['./app.component.scss']
 })
-export class AppComponent {
+export class AppComponent implements OnInit, OnDestroy {
 
-  constructor (private settings: SettingsService) {
+  newDocumentSubscription: Subscription;
+  successSubscription: Subscription;
+  failedSubscription: Subscription;
+
+  constructor (private settings: SettingsService, private consumerStatusService: ConsumerStatusService, private toastService: ToastService, private router: Router) {
     let anyWindow = (window as any)
     anyWindow.pdfWorkerSrc = '/assets/js/pdf.worker.min.js';
     this.settings.updateDarkModeSettings()
   }
 
+  ngOnDestroy(): void {
+    this.consumerStatusService.disconnect()
+    if (this.successSubscription) {
+      this.successSubscription.unsubscribe()
+    }
+    if (this.failedSubscription) {
+      this.failedSubscription.unsubscribe()
+    }
+    if (this.newDocumentSubscription) {
+      this.newDocumentSubscription.unsubscribe()
+    }
+  }
+
+  private showNotification(key) {
+    if (this.router.url == '/dashboard' && this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD)) {
+      return false
+    }
+    return this.settings.get(key)
+  }
+
+  ngOnInit(): void {
+    this.consumerStatusService.connect()
+
+    
+    this.successSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => {
+      if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS)) {
+        this.toastService.show({title: $localize`Document added`, delay: 10000, content: $localize`Document ${status.filename} was added to paperless.`, actionName: $localize`Open document`, action: () => {
+          this.router.navigate(['documents', status.documentId])
+        }})
+      }
+    })
+
+    this.failedSubscription = this.consumerStatusService.onDocumentConsumptionFailed().subscribe(status => {
+      if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED)) {
+        this.toastService.showError($localize`Could not add ${status.filename}\: ${status.message}`)
+      }
+    })
+
+    this.newDocumentSubscription = this.consumerStatusService.onDocumentDetected().subscribe(status => {
+      if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT)) {
+        this.toastService.show({title: $localize`New document detected`, delay: 5000, content: $localize`Document ${status.filename} is being processed by paperless.`})
+      }
+    })
+  }
+
 }

src-ui/src/app/app.module.ts

@@ -13,7 +13,7 @@ import { DocumentTypeListComponent } from './components/manage/document-type-lis
 import { LogsComponent } from './components/manage/logs/logs.component';
 import { SettingsComponent } from './components/manage/settings/settings.component';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
-import { DatePipe } from '@angular/common';
+import { DatePipe, registerLocaleData } from '@angular/common';
 import { NotFoundComponent } from './components/not-found/not-found.component';
 import { CorrespondentListComponent } from './components/manage/correspondent-list/correspondent-list.component';
 import { ConfirmDialogComponent } from './components/common/confirm-dialog/confirm-dialog.component';
@@ -59,6 +59,17 @@ import { SelectDialogComponent } from './components/common/select-dialog/select-
 import { NgSelectModule } from '@ng-select/ng-select';
 import { NumberComponent } from './components/common/input/number/number.component';
 import { SafePipe } from './pipes/safe.pipe';
+import { CustomDatePipe } from './pipes/custom-date.pipe';
+
+import localeFr from '@angular/common/locales/fr';
+import localeNl from '@angular/common/locales/nl';
+import localeDe from '@angular/common/locales/de';
+import localeEnGb from '@angular/common/locales/en-GB';
+
+registerLocaleData(localeFr)
+registerLocaleData(localeNl)
+registerLocaleData(localeDe)
+registerLocaleData(localeEnGb)
 
 @NgModule({
   declarations: [
@@ -108,7 +119,8 @@ import { SafePipe } from './pipes/safe.pipe';
     MetadataCollapseComponent,
     SelectDialogComponent,
     NumberComponent,
-    SafePipe
+    SafePipe,
+    CustomDatePipe
   ],
   imports: [
     BrowserModule,

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -52,12 +52,7 @@
 <div class="container-fluid">
   <div class="row">
     <nav id="sidebarMenu" class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse" [ngbCollapse]="isMenuCollapsed">
-
-      <div style="position: absolute; bottom: 0; left: 0;" class="text-muted p-1">
-        {{versionString}}
-      </div>
-
-      <div class="sidebar-sticky pt-3">
+      <div class="sidebar-sticky pt-3 d-flex flex-column justify-space-around">
         <ul class="nav flex-column">
           <li class="nav-item">
             <a class="nav-link" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()">
@@ -75,7 +70,7 @@
           </li>
         </ul>
 
-        <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted" *ngIf='savedViewService.sidebarViews.length > 0'>
+        <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='savedViewService.sidebarViews.length > 0'>
           <ng-container i18n>Saved views</ng-container>
         </h6>
         <ul class="nav flex-column mb-2">
@@ -88,7 +83,7 @@
           </li>
         </ul>
 
-        <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted" *ngIf='openDocuments.length > 0'>
+        <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='openDocuments.length > 0'>
           <ng-container i18n>Open documents</ng-container>
         </h6>
         <ul class="nav flex-column mb-2">
@@ -108,7 +103,7 @@
           </li>
         </ul>
 
-        <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
+        <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
           <ng-container i18n>Manage</ng-container>
         </h6>
         <ul class="nav flex-column mb-2">
@@ -156,8 +151,8 @@
           </li>
         </ul>
 
-        <h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
-          <ng-container i18n>Misc</ng-container>
+        <h6 class="sidebar-heading px-3 mt-auto pt-4 mb-1 text-muted">
+          <ng-container i18n>Info</ng-container>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item">
@@ -168,11 +163,24 @@
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://github.com/jonaswinkler/paperless-ng">
-              <svg class="sidebaricon" fill="currentColor">
-                <use xlink:href="assets/bootstrap-icons.svg#link"/>
-              </svg>&nbsp;<ng-container i18n>GitHub</ng-container>
-            </a>
+            <div class="d-flex w-100 flex-wrap">
+              <a class="nav-link pr-0 pb-0" target="_blank" rel="noopener noreferrer" href="https://github.com/jonaswinkler/paperless-ng">
+                <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="sidebaricon bi bi-github" viewBox="0 0 16 16">
+                  <path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.012 8.012 0 0 0 16 8c0-4.42-3.58-8-8-8z"/>
+                </svg>&nbsp;<ng-container i18n>GitHub</ng-container>
+              </a>
+              <a class="nav-link-additional small text-muted ml-3" target="_blank" rel="noopener noreferrer" href="https://github.com/jonaswinkler/paperless-ng/discussions/categories/feature-requests" title="Suggest an idea">
+                <svg xmlns="http://www.w3.org/2000/svg" width=".9rem" height=".9rem" fill="currentColor" class="bi bi-lightbulb pr-1" viewBox="0 0 16 16">
+                  <path d="M2 6a6 6 0 1 1 10.174 4.31c-.203.196-.359.4-.453.619l-.762 1.769A.5.5 0 0 1 10.5 13a.5.5 0 0 1 0 1 .5.5 0 0 1 0 1l-.224.447a1 1 0 0 1-.894.553H6.618a1 1 0 0 1-.894-.553L5.5 15a.5.5 0 0 1 0-1 .5.5 0 0 1 0-1 .5.5 0 0 1-.46-.302l-.761-1.77a1.964 1.964 0 0 0-.453-.618A5.984 5.984 0 0 1 2 6zm6-5a5 5 0 0 0-3.479 8.592c.263.254.514.564.676.941L5.83 12h4.342l.632-1.467c.162-.377.413-.687.676-.941A5 5 0 0 0 8 1z"/>
+                </svg>
+                <ng-container i18n>Suggest an idea</ng-container>
+              </a>
+            </div>
+          </li>
+          <li class="nav-item mt-2">
+            <div class="px-3 py-2 text-muted small">
+              {{versionString}}
+            </div>
           </li>
         </ul>
       </div>

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -24,6 +24,7 @@
   padding-top: 0.5rem;
   overflow-x: hidden;
   overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */
+  min-height: min-content;
 }
 @supports ((position: -webkit-sticky) or (position: sticky)) {
   .sidebar-sticky {
@@ -57,6 +58,20 @@
   text-transform: uppercase;
 }
 
+.nav {
+  flex-wrap: nowrap;
+}
+
+.nav-item .nav-link-additional {
+  margin-top: 0.2rem;
+  margin-left: 0.25rem;
+  padding-top: 0.5rem;
+
+  svg {
+    margin-bottom: 2px;
+  }
+}
+
 /*
  * Navbar
  */

src-ui/src/app/components/common/edit-dialog/edit-dialog.component.ts

@@ -27,6 +27,8 @@ export abstract class EditDialogComponent<T extends ObjectWithId> implements OnI
 
   networkActive = false
 
+  closeEnabled = false
+
   error = null
 
   abstract getForm(): FormGroup
@@ -37,6 +39,11 @@ export abstract class EditDialogComponent<T extends ObjectWithId> implements OnI
     if (this.object != null) {
       this.objectForm.patchValue(this.object)
     }
+
+    // wait to enable close button so it doesnt steal focus from input since its the first clickable element in the DOM
+    setTimeout(() => {
+      this.closeEnabled = true
+    });
   }
 
   getCreateTitle() {

src-ui/src/app/components/common/input/select/select.component.html

@@ -22,4 +22,12 @@
     </div>
   </div>
   <small *ngIf="hint" class="form-text text-muted">{{hint}}</small>
+  <small *ngIf="getSuggestions().length > 0">
+    <span i18n>Suggestions:</span>&nbsp;
+    <ng-container *ngFor="let s of getSuggestions()">
+      <a (click)="value = s.id; onChange(value)" [routerLink]="">{{s.name}}</a>&nbsp;
+    </ng-container>
+    
+    
+  </small>
 </div>

src-ui/src/app/components/common/input/select/select.component.ts

@@ -30,11 +30,22 @@ export class SelectComponent extends AbstractInputComponent<number> {
   @Input()
   allowNull: boolean = false
 
+  @Input()
+  suggestions: number[]
+
   @Output()
   createNew = new EventEmitter()
-  
+
   showPlusButton(): boolean {
     return this.createNew.observers.length > 0
   }
 
+  getSuggestions() {
+    if (this.suggestions && this.items) {
+      return this.suggestions.filter(id => id != this.value).map(id => this.items.find(item => item.id == id))
+    } else {
+      return []
+    }
+  }
+
 }

src-ui/src/app/components/common/input/tags/tags.component.html

@@ -2,30 +2,25 @@
   <label for="tags" i18n>Tags</label>
 
   <div class="input-group flex-nowrap">
-    <ng-select name="tags" [items]="tags" bindLabel="name" bindValue="id" [(ngModel)]="displayValue"
+    <ng-select name="tags" [items]="tags" bindLabel="name" bindValue="id" [(ngModel)]="value"
       [multiple]="true"
       [closeOnSelect]="false"
       [clearSearchOnAdd]="true"
-      [disabled]="disabled"
       [hideSelected]="true"
-      (change)="ngSelectChange()">
+      (change)="onChange(value)"
+      (blur)="onTouched()">
 
       <ng-template ng-label-tmp let-item="item">
         <span class="tag-wrap tag-wrap-delete" (click)="removeTag(item.id)">
           <svg width="1.2em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
             <use xlink:href="assets/bootstrap-icons.svg#x"/>
           </svg>
-          <app-tag style="background-color: none;" [tag]="getTag(item.id)"></app-tag>
+          <app-tag *ngIf="item.id && tags" style="background-color: none;" [tag]="getTag(item.id)"></app-tag>
         </span>
       </ng-template>
       <ng-template ng-option-tmp let-item="item" let-index="index" let-search="searchTerm">
         <div class="tag-wrap">
-          <div class="selected-icon d-inline-block mr-1">
-            <svg *ngIf="displayValue.includes(item.id)" width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
-              <use xlink:href="assets/bootstrap-icons.svg#check"/>
-            </svg>
-          </div>
-          <app-tag class="mr-2" [tag]="getTag(item.id)"></app-tag>
+          <app-tag *ngIf="item.id && tags" class="mr-2" [tag]="getTag(item.id)"></app-tag>
         </div>
       </ng-template>
     </ng-select>
@@ -39,5 +34,13 @@
     </div>
   </div>
   <small class="form-text text-muted" *ngIf="hint">{{hint}}</small>
+  <small *ngIf="getSuggestions().length > 0">
+    <span i18n>Suggestions:</span>&nbsp;
+    <ng-container *ngFor="let tag of getSuggestions()">
+      <a (click)="addTag(tag.id)" [routerLink]="">{{tag.name}}</a>&nbsp;
+    </ng-container>
+    
+    
+  </small>
 
 </div>

src-ui/src/app/components/common/input/tags/tags.component.ts

@@ -26,9 +26,6 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
 
   writeValue(newValue: number[]): void {
     this.value = newValue
-    if (this.tags) {
-      this.displayValue = newValue
-    }
   }
   registerOnChange(fn: any): void {
     this.onChange = fn;
@@ -43,7 +40,6 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
   ngOnInit(): void {
     this.tagService.listAll().subscribe(result => {
       this.tags = result.results
-      this.displayValue = this.value
     })
   }
 
@@ -53,23 +49,28 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
   @Input()
   hint
 
-  value: number[]
+  @Input()
+  suggestions: number[]
 
-  displayValue: number[] = []
+  value: number[]
 
   tags: PaperlessTag[]
 
   getTag(id) {
-    return this.tags.find(tag => tag.id == id)
+    if (this.tags) {
+      return this.tags.find(tag => tag.id == id)
+    } else {
+      return null
+    }
   }
 
   removeTag(id) {
-    let index = this.displayValue.indexOf(id)
+    let index = this.value.indexOf(id)
     if (index > -1) {
-      let oldValue = this.displayValue
+      let oldValue = this.value
       oldValue.splice(index, 1)
-      this.displayValue = [...oldValue]
-      this.onChange(this.displayValue)
+      this.value = [...oldValue]
+      this.onChange(this.value)
     }
   }
 
@@ -79,15 +80,23 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
     modal.componentInstance.success.subscribe(newTag => {
       this.tagService.listAll().subscribe(tags => {
         this.tags = tags.results
-        this.displayValue = [...this.displayValue, newTag.id]
-        this.onChange(this.displayValue)
+        this.value = [...this.value, newTag.id]
+        this.onChange(this.value)
       })
     })
   }
 
-  ngSelectChange() {
-    this.value = this.displayValue
-    this.onChange(this.displayValue)
+  getSuggestions() {
+    if (this.suggestions && this.tags) {
+      return this.suggestions.filter(id => !this.value.includes(id)).map(id => this.tags.find(tag => tag.id == id))
+    } else {
+      return []
+    }
+  }
+
+  addTag(id) {
+    this.value = [...this.value, id]
+    this.onChange(this.value)
   }
 
 }

src-ui/src/app/components/common/toasts/toasts.component.html

@@ -3,5 +3,6 @@
   [header]="toast.title" [autohide]="true" [delay]="toast.delay"
   [class]="toast.classname"
   (hide)="toastService.closeToast(toast)">
-  {{toast.content}}
-</ngb-toast>
+  <p>{{toast.content}}</p>
+  <p *ngIf="toast.action"><button class="btn btn-sm btn-outline-secondary" (click)="toastService.closeToast(toast); toast.action()">{{toast.actionName}}</button></p>
+</ngb-toast>

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html

@@ -12,10 +12,10 @@
     </thead>
     <tbody>
       <tr *ngFor="let doc of documents" routerLink="/documents/{{doc.id}}">
-        <td>{{doc.created | date}}</td>
+        <td>{{doc.created | customDate}}</td>
         <td>{{doc.title | documentTitle}}<app-tag [tag]="t" *ngFor="let t of doc.tags$ | async" class="ml-1"></app-tag>
       </tr>
     </tbody>
   </table>
 
-</app-widget-frame>
+</app-widget-frame>

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts

@@ -1,8 +1,10 @@
-import { Component, Input, OnInit } from '@angular/core';
+import { Component, Input, OnDestroy, OnInit } from '@angular/core';
 import { Router } from '@angular/router';
+import { Subscription } from 'rxjs';
 import { PaperlessDocument } from 'src/app/data/paperless-document';
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view';
 import { DocumentListViewService } from 'src/app/services/document-list-view.service';
+import { ConsumerStatusService } from 'src/app/services/consumer-status.service';
 import { DocumentService } from 'src/app/services/rest/document.service';
 
 @Component({
@@ -10,19 +12,33 @@ import { DocumentService } from 'src/app/services/rest/document.service';
   templateUrl: './saved-view-widget.component.html',
   styleUrls: ['./saved-view-widget.component.scss']
 })
-export class SavedViewWidgetComponent implements OnInit {
+export class SavedViewWidgetComponent implements OnInit, OnDestroy {
 
   constructor(
     private documentService: DocumentService,
     private router: Router,
-    private list: DocumentListViewService) { }
-  
+    private list: DocumentListViewService,
+    private consumerStatusService: ConsumerStatusService) { }
+
   @Input()
   savedView: PaperlessSavedView
 
   documents: PaperlessDocument[] = []
 
+  subscription: Subscription
+
   ngOnInit(): void {
+    this.reload()
+    this.subscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => {
+      this.reload()
+    })
+  }
+
+  ngOnDestroy(): void {
+    this.subscription.unsubscribe()
+  }
+
+  reload() {
     this.documentService.listFiltered(1,10,this.savedView.sort_field, this.savedView.sort_reverse, this.savedView.filter_rules).subscribe(result => {
       this.documents = result.results
     })
@@ -32,9 +48,9 @@ export class SavedViewWidgetComponent implements OnInit {
     if (this.savedView.show_in_sidebar) {
       this.router.navigate(['view', this.savedView.id])
     } else {
-      this.list.load(this.savedView)
+      this.list.loadSavedView(this.savedView, true)
       this.router.navigate(["documents"])
-      }
+    }
   }
 
 }

src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html

@@ -1,6 +1,6 @@
 <app-widget-frame title="Statistics" i18n-title>
   <ng-container content>
-    <p class="card-text" i18n>Documents in inbox: {{statistics.documents_inbox}}</p>
-    <p class="card-text" i18n>Total documents: {{statistics.documents_total}}</p>
+    <p class="card-text" i18n *ngIf="statistics?.documents_inbox != null">Documents in inbox: {{statistics?.documents_inbox}}</p>
+    <p class="card-text" i18n>Total documents: {{statistics?.documents_total}}</p>
   </ng-container>
-</app-widget-frame>
+</app-widget-frame>

src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts

@@ -1,6 +1,7 @@
 import { HttpClient } from '@angular/common/http';
-import { Component, OnInit } from '@angular/core';
-import { Observable } from 'rxjs';
+import { Component, OnDestroy, OnInit } from '@angular/core';
+import { Observable, Subscription } from 'rxjs';
+import { ConsumerStatusService } from 'src/app/services/consumer-status.service';
 import { environment } from 'src/environments/environment';
 
 export interface Statistics {
@@ -14,20 +15,34 @@ export interface Statistics {
   templateUrl: './statistics-widget.component.html',
   styleUrls: ['./statistics-widget.component.scss']
 })
-export class StatisticsWidgetComponent implements OnInit {
+export class StatisticsWidgetComponent implements OnInit, OnDestroy {
 
-  constructor(private http: HttpClient) { }
+  constructor(private http: HttpClient,
+    private consumerStatusService: ConsumerStatusService) { }
 
   statistics: Statistics = {}
 
-  getStatistics(): Observable<Statistics> {
+  subscription: Subscription
+  
+  private getStatistics(): Observable<Statistics> {
     return this.http.get(`${environment.apiBaseUrl}statistics/`)
   }
-  
-  ngOnInit(): void {
+
+  reload() {
     this.getStatistics().subscribe(statistics => {
       this.statistics = statistics
     })
   }
 
+  ngOnInit(): void {
+    this.reload()
+    this.subscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => {
+      this.reload()
+    })
+  }
+
+  ngOnDestroy(): void {
+    this.subscription.unsubscribe()
+  }
+
 }

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html

@@ -1,18 +1,52 @@
 <app-widget-frame title="Upload new documents" i18n-title>
-
+  <div header-buttons>
+    <a *ngIf="getStatusSuccess().length > 0" (click)="dismissCompleted()" [routerLink]="" >
+      <span i18n="This button dismisses all status messages about processed documents on the dashboard (failed and successful)">Dismiss completed</span>&nbsp;
+      <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-check2-all" viewBox="0 0 16 16">
+        <path d="M12.354 4.354a.5.5 0 0 0-.708-.708L5 10.293 1.854 7.146a.5.5 0 1 0-.708.708l3.5 3.5a.5.5 0 0 0 .708 0l7-7zm-4.208 7l-.896-.897.707-.707.543.543 6.646-6.647a.5.5 0 0 1 .708.708l-7 7a.5.5 0 0 1-.708 0z"/>
+        <path d="M5.354 7.146l.896.897-.707.707-.897-.896a.5.5 0 1 1 .708-.708z"/>
+      </svg>
+    </a>
+  </div>
   <div content>
     <form>
       <ngx-file-drop dropZoneLabel="Drop documents here or" browseBtnLabel="Browse files" (onFileDrop)="dropped($event)"
         (onFileOver)="fileOver($event)" (onFileLeave)="fileLeave($event)" dropZoneClassName="bg-light card"
-        multiple="true" contentClassName="justify-content-center d-flex align-items-center p-5" [showBrowseBtn]=true
+        multiple="true" contentClassName="justify-content-center d-flex align-items-center py-5 px-2" [showBrowseBtn]=true
         browseBtnClassName="btn btn-sm btn-outline-primary ml-2" i18n-dropZoneLabel i18n-browseBtnLabel>
-
       </ngx-file-drop>
     </form>
-    <div *ngIf="uploadVisible" class="mt-3">
-      <p i18n>{uploadStatus.length, plural, =1 {Uploading file...} =other {Uploading {{uploadStatus.length}} files...}}</p>
-      <ngb-progressbar [value]="loadedSum" [max]="totalSum" [striped]="true" [animated]="uploadStatus.length > 0">
-      </ngb-progressbar>
+    <p class="mt-3" *ngIf="getStatus().length > 0">{{getStatusSummary()}}</p>
+    <div *ngFor="let status of getStatus()">
+      <ng-container [ngTemplateOutlet]="consumerAlert" [ngTemplateOutletContext]="{ $implicit: status }"></ng-container>
+    </div>
+    <div *ngIf="getStatusHidden().length" class="alerts-hidden">
+      <p *ngIf="!alertsExpanded" class="mt-3 mb-0 text-center">
+        <span i18n="This is shown as a summary line when there are more than 5 document in the processing pipeline.">{getStatusHidden().length, plural, =1 {One more document} other {{{getStatusHidden().length}} more documents}}</span>
+        &nbsp;&bull;&nbsp;
+        <a [routerLink]="" (click)="alertsExpanded = !alertsExpanded" aria-controls="hiddenAlerts" [attr.aria-expanded]="alertsExpanded" i18n>Show all</a>
+      </p>
+      <div #hiddenAlerts="ngbCollapse" [(ngbCollapse)]="!alertsExpanded">
+        <div *ngFor="let status of getStatusHidden()">
+          <ng-container [ngTemplateOutlet]="consumerAlert" [ngTemplateOutletContext]="{ $implicit: status }"></ng-container>
+        </div>
+      </div>
     </div>
   </div>
-</app-widget-frame>
+</app-widget-frame>
+
+<ng-template #consumerAlert let-status>
+  <ngb-alert type="secondary" class="mt-2 mb-0" [dismissible]="isFinished(status)" (closed)="dismiss(status)">
+    <h6 class="alert-heading">{{status.filename}}</h6>
+    <p class="mb-0 pb-1" *ngIf="!isFinished(status) || (isFinished(status) && !status.documentId)">{{status.message}}</p>
+    <ngb-progressbar [value]="status.getProgress()" [max]="1" [type]="getStatusColor(status)"></ngb-progressbar>
+    <div *ngIf="isFinished(status)">
+      <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary btn-open" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">
+        <small i18n>Open document</small>
+        <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
+          <path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
+        </svg>
+      </button>
+    </div>
+  </ngb-alert>
+</ng-template>

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss

@@ -0,0 +1,35 @@
+@import "/src/theme";
+
+form {
+  position: relative;
+}
+
+.alert-heading {
+  font-size: 80%;
+  font-weight: bold;
+}
+
+.alerts-hidden {
+  .btn {
+    line-height: 1;
+  }
+}
+
+.btn-open {
+  line-height: 1;
+
+  svg {
+    margin-top: -1px;
+  }
+}
+
+::ng-deep .progress {
+  position: absolute;
+  top: 0;
+  right: 0;
+  bottom: 0;
+  left: 0;
+  height: auto;
+  mix-blend-mode: soft-light;
+  pointer-events: none;
+}

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts

@@ -1,14 +1,10 @@
 import { HttpEventType } from '@angular/common/http';
 import { Component, OnInit } from '@angular/core';
 import { FileSystemFileEntry, NgxFileDropEntry } from 'ngx-file-drop';
+import { ConsumerStatusService, FileStatus, FileStatusPhase } from 'src/app/services/consumer-status.service';
 import { DocumentService } from 'src/app/services/rest/document.service';
-import { ToastService } from 'src/app/services/toast.service';
 
-
-interface UploadStatus {
-  loaded: number
-  total: number 
-}
+const MAX_ALERTS = 5
 
 @Component({
   selector: 'app-upload-file-widget',
@@ -16,8 +12,89 @@ interface UploadStatus {
   styleUrls: ['./upload-file-widget.component.scss']
 })
 export class UploadFileWidgetComponent implements OnInit {
+  alertsExpanded = false
 
-  constructor(private documentService: DocumentService, private toastService: ToastService) { }
+  constructor(
+    private documentService: DocumentService,
+    private consumerStatusService: ConsumerStatusService
+  ) { }
+
+  getStatus() {
+    return this.consumerStatusService.getConsumerStatus().slice(0, MAX_ALERTS)
+  }
+
+  getStatusSummary() {
+    let strings = []
+    let countUploadingAndProcessing =  this.consumerStatusService.getConsumerStatusNotCompleted().length
+    let countFailed = this.getStatusFailed().length
+    let countSuccess = this.getStatusSuccess().length
+    if (countUploadingAndProcessing > 0) {
+      strings.push($localize`Processing: ${countUploadingAndProcessing}`)
+    }
+    if (countFailed > 0) {
+      strings.push($localize`Failed: ${countFailed}`)
+    }
+    if (countSuccess > 0) {
+      strings.push($localize`Added: ${countSuccess}`)
+    }
+    return strings.join($localize`:this string is used to separate processing, failed and added on the file upload widget:, `)
+  }
+
+  getStatusHidden() {
+    if (this.consumerStatusService.getConsumerStatus().length < MAX_ALERTS) return []
+    else return this.consumerStatusService.getConsumerStatus().slice(MAX_ALERTS)
+  }
+
+  getStatusUploading() {
+    return this.consumerStatusService.getConsumerStatus(FileStatusPhase.UPLOADING)
+  }
+
+  getStatusFailed() {
+    return this.consumerStatusService.getConsumerStatus(FileStatusPhase.FAILED)
+  }
+
+  getStatusSuccess() {
+    return this.consumerStatusService.getConsumerStatus(FileStatusPhase.SUCCESS)
+  }
+
+  getStatusCompleted() {
+    return this.consumerStatusService.getConsumerStatusCompleted()
+  }
+  getTotalUploadProgress() {
+    let current = 0
+    let max = 0
+
+    this.getStatusUploading().forEach(status => {
+      current += status.currentPhaseProgress
+      max += status.currentPhaseMaxProgress
+    })
+
+    return current / Math.max(max, 1)
+  }
+
+  isFinished(status: FileStatus) {
+    return status.phase == FileStatusPhase.FAILED || status.phase == FileStatusPhase.SUCCESS
+  }
+
+  getStatusColor(status: FileStatus) {
+    switch (status.phase) {
+      case FileStatusPhase.PROCESSING:
+      case FileStatusPhase.UPLOADING:
+          return "primary"
+      case FileStatusPhase.FAILED:
+        return "danger"
+      case FileStatusPhase.SUCCESS:
+        return "success"
+    }
+  }
+
+  dismiss(status: FileStatus) {
+    this.consumerStatusService.dismiss(status)
+  }
+
+  dismissCompleted() {
+    this.consumerStatusService.dismissCompleted()
+  }
 
   ngOnInit(): void {
   }
@@ -28,54 +105,39 @@ export class UploadFileWidgetComponent implements OnInit {
   public fileLeave(event){
   }
 
-  uploadStatus: UploadStatus[] = []
-  completedFiles = 0
-
-  uploadVisible = false
-
-  get loadedSum() {
-    return this.uploadStatus.map(s => s.loaded).reduce((a,b) => a+b, this.completedFiles > 0 ? 1 : 0)
-  }
-
-  get totalSum() {
-    return this.uploadStatus.map(s => s.total).reduce((a,b) => a+b, 1)
-  }
-
   public dropped(files: NgxFileDropEntry[]) {
     for (const droppedFile of files) {
       if (droppedFile.fileEntry.isFile) {
-      let uploadStatusObject: UploadStatus = {loaded: 0, total: 1}
-      this.uploadStatus.push(uploadStatusObject)
-      this.uploadVisible = true
 
       const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
         fileEntry.file((file: File) => {
           let formData = new FormData()
           formData.append('document', file, file.name)
+          let status = this.consumerStatusService.newFileUpload(file.name)
+
+          status.message = $localize`Connecting...`
 
           this.documentService.uploadDocument(formData).subscribe(event => {
             if (event.type == HttpEventType.UploadProgress) {
-              uploadStatusObject.loaded = event.loaded
-              uploadStatusObject.total = event.total
+              status.updateProgress(FileStatusPhase.UPLOADING, event.loaded, event.total)
+              status.message = $localize`Uploading...`
             } else if (event.type == HttpEventType.Response) {
-              this.uploadStatus.splice(this.uploadStatus.indexOf(uploadStatusObject), 1)
-              this.completedFiles += 1
-              this.toastService.showInfo($localize`The document has been uploaded and will be processed by the consumer shortly.`)
+              status.taskId = event.body["task_id"]
+              status.message = $localize`Upload complete, waiting...`
             }
-            
+
           }, error => {
-            this.uploadStatus.splice(this.uploadStatus.indexOf(uploadStatusObject), 1)
-            this.completedFiles += 1
             switch (error.status) {
               case 400: {
-                this.toastService.showInfo($localize`There was an error while uploading the document: ${error.error.document}`)
+                this.consumerStatusService.fail(status, error.error.document)
                 break;
               }
               default: {
-                this.toastService.showInfo($localize`An error has occurred while uploading the document. Sorry!`)
+                this.consumerStatusService.fail(status, $localize`HTTP error: ${error.status} ${error.statusText}`)
                 break;
               }
             }
+
           })
         });
       }

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -60,10 +60,10 @@
                         <app-input-number i18n-title title="Archive serial number" [error]="error?.archive_serial_number" formControlName='archive_serial_number'></app-input-number>
                         <app-input-date-time i18n-titleDate titleDate="Date created" formControlName="created"></app-input-date-time>
                         <app-input-select [items]="correspondents" i18n-title title="Correspondent" formControlName="correspondent" [allowNull]="true"
-                            (createNew)="createCorrespondent()"></app-input-select>
+                            (createNew)="createCorrespondent()" [suggestions]="suggestions?.correspondents"></app-input-select>
                         <app-input-select [items]="documentTypes" i18n-title title="Document type" formControlName="document_type" [allowNull]="true"
-                            (createNew)="createDocumentType()"></app-input-select>
-                        <app-input-tags formControlName="tags"></app-input-tags>
+                            (createNew)="createDocumentType()" [suggestions]="suggestions?.document_types"></app-input-select>
+                        <app-input-tags formControlName="tags" [suggestions]="suggestions?.tags"></app-input-tags>
 
                     </ng-template>
                 </li>
@@ -85,11 +85,11 @@
                             <tbody>
                                 <tr>
                                     <td i18n>Date modified</td>
-                                    <td>{{document.modified | date:'medium'}}</td>
+                                    <td>{{document.modified | customDate}}</td>
                                 </tr>
                                 <tr>
                                     <td i18n>Date added</td>
-                                    <td>{{document.added | date:'medium'}}</td>
+                                    <td>{{document.added | customDate}}</td>
                                 </tr>
                                 <tr>
                                     <td i18n>Media filename</td>
@@ -141,6 +141,10 @@
             <ng-template #nativePdfViewer>
                 <object [data]="previewUrl | safe" type="application/pdf" class="preview-sticky" width="100%"></object>
             </ng-template>
-        </ng-container>     
+        </ng-container>
+        <ng-container *ngIf="getContentType() == 'text/plain'">
+            <object [data]="previewUrl | safe" type="text/plain" class="preview-sticky" width="100%"></object>
+        </ng-container>
+
     </div>
 </div>

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -19,6 +19,7 @@ import { PDFDocumentProxy } from 'ng2-pdf-viewer';
 import { ToastService } from 'src/app/services/toast.service';
 import { TextComponent } from '../common/input/text/text.component';
 import { SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
+import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions';
 
 @Component({
   selector: 'app-document-detail',
@@ -40,6 +41,8 @@ export class DocumentDetailComponent implements OnInit {
   documentId: number
   document: PaperlessDocument
   metadata: PaperlessDocumentMetadata
+  suggestions: PaperlessDocumentSuggestions
+
   title: string
   previewUrl: string
   downloadUrl: string
@@ -95,6 +98,7 @@ export class DocumentDetailComponent implements OnInit {
       this.previewUrl = this.documentsService.getPreviewUrl(this.documentId)
       this.downloadUrl = this.documentsService.getDownloadUrl(this.documentId)
       this.downloadOriginalUrl = this.documentsService.getDownloadUrl(this.documentId, true)
+      this.suggestions = null
       if (this.openDocumentService.getOpenDocument(this.documentId)) {
         this.updateComponent(this.openDocumentService.getOpenDocument(this.documentId))
       } else {
@@ -111,6 +115,13 @@ export class DocumentDetailComponent implements OnInit {
     this.document = doc
     this.documentsService.getMetadata(doc.id).subscribe(result => {
       this.metadata = result
+    }, error => {
+      this.metadata = null
+    })
+    this.documentsService.getSuggestions(doc.id).subscribe(result => {
+      this.suggestions = result
+    }, error => {
+      this.suggestions = null
     })
     this.title = this.documentTitlePipe.transform(doc.title)
     this.documentForm.patchValue(doc)
@@ -180,8 +191,8 @@ export class DocumentDetailComponent implements OnInit {
 
   close() {
     this.openDocumentService.closeDocument(this.document)
-    if (this.documentListViewService.savedViewId) {
-      this.router.navigate(['view', this.documentListViewService.savedViewId])
+    if (this.documentListViewService.activeSavedViewId) {
+      this.router.navigate(['view', this.documentListViewService.activeSavedViewId])
     } else {
       this.router.navigate(['documents'])
     }

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html

@@ -56,6 +56,20 @@
     </div>
   </div>
   <div class="col-auto ml-auto mb-2 mb-xl-0 d-flex">
+    <div class="btn-group btn-group-sm mr-2">
+      <button type="button" class="btn btn-outline-primary btn-sm" (click)="downloadSelected()">
+        <svg width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor">
+          <use xlink:href="assets/bootstrap-icons.svg#download" />
+        </svg>&nbsp;<ng-container i18n>Download</ng-container>
+      </button>
+      <div class="btn-group" ngbDropdown role="group" aria-label="Button group with nested dropdown">
+        <button class="btn btn-outline-primary btn-sm dropdown-toggle-split" ngbDropdownToggle></button>
+        <div class="dropdown-menu shadow" ngbDropdownMenu>
+          <button ngbDropdownItem i18n (click)="downloadSelected('originals')">Download originals</button>
+        </div>
+      </div>
+    </div>
+
     <button type="button" class="btn btn-sm btn-outline-danger" (click)="applyDelete()">
       <svg width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor">
         <use xlink:href="assets/bootstrap-icons.svg#trash" />

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts

@@ -15,6 +15,7 @@ import { ToggleableItemState } from '../../common/filterable-dropdown/toggleable
 import { MatchingModel } from 'src/app/data/matching-model';
 import { SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
 import { ToastService } from 'src/app/services/toast.service';
+import { saveAs } from 'file-saver';
 
 @Component({
   selector: 'app-bulk-editor',
@@ -109,7 +110,7 @@ export class BulkEditorComponent {
     if (items.length == 0) {
       return ""
     } else if (items.length == 1) {
-      return items[0].name
+      return $localize`"${items[0].name}"`
     } else if (items.length == 2) {
       return $localize`:This is for messages like 'modify "tag1" and "tag2"':"${items[0].name}" and "${items[1].name}"`
     } else {
@@ -137,7 +138,7 @@ export class BulkEditorComponent {
       } else {
         modal.componentInstance.message = $localize`This operation will add the tags ${this._localizeList(changedTags.itemsToAdd)} and remove the tags ${this._localizeList(changedTags.itemsToRemove)} on ${this.list.selected.size} selected document(s).`
       }
-      
+
       modal.componentInstance.btnClass = "btn-warning"
       modal.componentInstance.btnCaption = $localize`Confirm`
       modal.componentInstance.confirmClicked.subscribe(() => {
@@ -207,4 +208,10 @@ export class BulkEditorComponent {
       this.executeBulkOperation(modal, "delete", {})
     })
   }
+
+  downloadSelected(content = "archive") {
+    this.documentService.bulkDownload(Array.from(this.list.selected), content).subscribe((result: any) => {
+      saveAs(result, 'documents.zip');
+    })
+  }
 }

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html

@@ -1,11 +1,11 @@
 <div class="card mb-3 shadow-sm" [class.card-selected]="selected" [class.document-card]="selectable">
   <div class="row no-gutters">
-    <div class="col-md-2 d-none d-lg-block doc-img-background rounded-left" [class.doc-img-background-selected]="selected">
-      <img [src]="getThumbUrl()" class="card-img doc-img border-right rounded-left" (click)="setSelected(selectable ? !selected : false)">
+    <div class="col-md-2 d-none d-lg-block doc-img-background rounded-left" [class.doc-img-background-selected]="selected" (click)="this.toggleSelected.emit($event)">
+      <img [src]="getThumbUrl()" class="card-img doc-img border-right rounded-left">
 
       <div style="top: 0; left: 0" class="position-absolute border-right border-bottom bg-light p-1" [class.document-card-check]="!selected">
         <div class="custom-control custom-checkbox">
-          <input type="checkbox" class="custom-control-input" id="smallCardCheck{{document.id}}" [checked]="selected" (change)="setSelected($event.target.checked)">
+          <input type="checkbox" class="custom-control-input" id="smallCardCheck{{document.id}}" [checked]="selected" (click)="this.toggleSelected.emit($event)">
           <label class="custom-control-label" for="smallCardCheck{{document.id}}"></label>
         </div>
       </div>
@@ -17,11 +17,11 @@
         <div class="d-flex justify-content-between align-items-center">
           <h5 class="card-title">
             <ng-container *ngIf="document.correspondent">
-              <a *ngIf="clickCorrespondent.observers.length ; else nolink" [routerLink]="" title="Filter by correspondent" i18n-title (click)="clickCorrespondent.emit(document.correspondent)" class="font-weight-bold">{{(document.correspondent$ | async)?.name}}</a>
+              <a *ngIf="clickCorrespondent.observers.length ; else nolink" [routerLink]="" title="Filter by correspondent" i18n-title (click)="clickCorrespondent.emit(document.correspondent);$event.stopPropagation()" class="font-weight-bold">{{(document.correspondent$ | async)?.name}}</a>
               <ng-template #nolink>{{(document.correspondent$ | async)?.name}}</ng-template>:
             </ng-container>
             {{document.title | documentTitle}}
-            <app-tag [tag]="t" linkTitle="Filter by tag" i18n-linkTitle *ngFor="let t of document.tags$ | async" class="ml-1" (click)="clickTag.emit(t.id)" [clickable]="clickTag.observers.length"></app-tag>
+            <app-tag [tag]="t" linkTitle="Filter by tag" i18n-linkTitle *ngFor="let t of document.tags$ | async" class="ml-1" (click)="clickTag.emit(t.id);$event.stopPropagation()" [clickable]="clickTag.observers.length"></app-tag>
           </h5>
           <h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archive_serial_number}}</h5>
         </div>
@@ -44,9 +44,9 @@
               </svg>&nbsp;<span class="d-block d-md-inline" i18n>Edit</span>
             </a>
             <a class="btn btn-sm btn-outline-secondary" [href]="getPreviewUrl()">
-              <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-search" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
-                <path fill-rule="evenodd" d="M10.442 10.442a1 1 0 0 1 1.415 0l3.85 3.85a1 1 0 0 1-1.414 1.415l-3.85-3.85a1 1 0 0 1 0-1.415z"/>
-                <path fill-rule="evenodd" d="M6.5 12a5.5 5.5 0 1 0 0-11 5.5 5.5 0 0 0 0 11zM13 6.5a6.5 6.5 0 1 1-13 0 6.5 6.5 0 0 1 13 0z"/>
+              <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-eye" viewBox="0 0 16 16">
+                <path d="M16 8s-3-5.5-8-5.5S0 8 0 8s3 5.5 8 5.5S16 8 16 8zM1.173 8a13.133 13.133 0 0 1 1.66-2.043C4.12 4.668 5.88 3.5 8 3.5c2.12 0 3.879 1.168 5.168 2.457A13.133 13.133 0 0 1 14.828 8c-.058.087-.122.183-.195.288-.335.48-.83 1.12-1.465 1.755C11.879 11.332 10.119 12.5 8 12.5c-2.12 0-3.879-1.168-5.168-2.457A13.134 13.134 0 0 1 1.172 8z"/>
+                <path d="M8 5.5a2.5 2.5 0 1 0 0 5 2.5 2.5 0 0 0 0-5zM4.5 8a3.5 3.5 0 1 1 7 0 3.5 3.5 0 0 1-7 0z"/>
               </svg>&nbsp;<span class="d-block d-md-inline" i18n>View</span>
             </a>
             <a class="btn btn-sm btn-outline-secondary" [href]="getDownloadUrl()">
@@ -64,7 +64,7 @@
             <ngb-progressbar [type]="searchScoreClass" [value]="searchScore" class="search-score-bar mx-2" [max]="1"></ngb-progressbar>
           </div>
 
-          <small class="text-muted" [class.ml-auto]="!searchScore" i18n>Created: {{document.created | date}}</small>
+          <small class="text-muted" [class.ml-auto]="!searchScore" i18n>Created: {{document.created | customDate}}</small>
         </div>
 
       </div>

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss

@@ -6,7 +6,7 @@
 
 .doc-img {
   object-fit: cover;
-  object-position: top;
+  object-position: top left;
   height: 100%;
   position: absolute;
   mix-blend-mode: multiply;

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts

@@ -15,16 +15,11 @@ export class DocumentCardLargeComponent implements OnInit {
   @Input()
   selected = false
 
-  setSelected(value: boolean) {
-    this.selected = value
-    this.selectedChange.emit(value)
-  }
-
   @Output()
-  selectedChange = new EventEmitter<boolean>()
+  toggleSelected = new EventEmitter()
 
   get selectable() {
-    return this.selectedChange.observers.length > 0
+    return this.toggleSelected.observers.length > 0
   }
 
   @Input()

src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html

@@ -1,18 +1,18 @@
 <div class="col p-2 h-100">
   <div class="card h-100 shadow-sm document-card" [class.card-selected]="selected">
-    <div class="border-bottom doc-img-container" [class.doc-img-background-selected]="selected">
-      <img class="card-img doc-img rounded-top" [src]="getThumbUrl()" (click)="setSelected(!selected)">
+    <div class="border-bottom doc-img-container" [class.doc-img-background-selected]="selected" (click)="this.toggleSelected.emit($event)">
+      <img class="card-img doc-img rounded-top" [src]="getThumbUrl()">
 
       <div class="border-right border-bottom bg-light p-1 rounded document-card-check">
         <div class="custom-control custom-checkbox">
-          <input type="checkbox" class="custom-control-input" id="smallCardCheck{{document.id}}" [checked]="selected" (change)="setSelected($event.target.checked)">
+          <input type="checkbox" class="custom-control-input" id="smallCardCheck{{document.id}}" [checked]="selected" (click)="this.toggleSelected.emit($event)">
           <label class="custom-control-label" for="smallCardCheck{{document.id}}"></label>
         </div>
       </div>
 
       <div style="top: 0; right: 0; font-size: large" class="text-right position-absolute mr-1">
         <div *ngFor="let t of getTagsLimited$() | async">
-          <app-tag [tag]="t" (click)="clickTag.emit(t.id)" [clickable]="true" linkTitle="Filter by tag" i18n-linkTitle></app-tag>
+          <app-tag [tag]="t" (click)="clickTag.emit(t.id);$event.stopPropagation()" [clickable]="true" linkTitle="Filter by tag" i18n-linkTitle></app-tag>
         </div>
         <div *ngIf="moreTags">
           <span class="badge badge-secondary">+ {{moreTags}}</span>
@@ -23,7 +23,7 @@
     <div class="card-body p-2">
       <p class="card-text">
         <ng-container *ngIf="document.correspondent">
-          <a [routerLink]="" title="Filter by correspondent" i18n-title (click)="clickCorrespondent.emit(document.correspondent)" class="font-weight-bold">{{(document.correspondent$ | async)?.name}}</a>:
+          <a [routerLink]="" title="Filter by correspondent" i18n-title (click)="clickCorrespondent.emit(document.correspondent);$event.stopPropagation()" class="font-weight-bold">{{(document.correspondent$ | async)?.name}}</a>:
         </ng-container>
         {{document.title | documentTitle}} <span *ngIf="document.archive_serial_number">(#{{document.archive_serial_number}})</span>
       </p>
@@ -38,19 +38,19 @@
             </svg>
           </a>
           <a [href]="getPreviewUrl()" class="btn btn-sm btn-outline-secondary" title="View in browser" i18n-title>
-            <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-search" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
-              <path fill-rule="evenodd" d="M10.442 10.442a1 1 0 0 1 1.415 0l3.85 3.85a1 1 0 0 1-1.414 1.415l-3.85-3.85a1 1 0 0 1 0-1.415z"/>
-              <path fill-rule="evenodd" d="M6.5 12a5.5 5.5 0 1 0 0-11 5.5 5.5 0 0 0 0 11zM13 6.5a6.5 6.5 0 1 1-13 0 6.5 6.5 0 0 1 13 0z"/>
+            <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-eye" viewBox="0 0 16 16">
+              <path d="M16 8s-3-5.5-8-5.5S0 8 0 8s3 5.5 8 5.5S16 8 16 8zM1.173 8a13.133 13.133 0 0 1 1.66-2.043C4.12 4.668 5.88 3.5 8 3.5c2.12 0 3.879 1.168 5.168 2.457A13.133 13.133 0 0 1 14.828 8c-.058.087-.122.183-.195.288-.335.48-.83 1.12-1.465 1.755C11.879 11.332 10.119 12.5 8 12.5c-2.12 0-3.879-1.168-5.168-2.457A13.134 13.134 0 0 1 1.172 8z"/>
+              <path d="M8 5.5a2.5 2.5 0 1 0 0 5 2.5 2.5 0 0 0 0-5zM4.5 8a3.5 3.5 0 1 1 7 0 3.5 3.5 0 0 1-7 0z"/>
             </svg>
           </a>
-          <a [href]="getDownloadUrl()" class="btn btn-sm btn-outline-secondary" title="Download" i18n-title>
+          <a [href]="getDownloadUrl()" class="btn btn-sm btn-outline-secondary" title="Download" (click)="$event.stopPropagation()" i18n-title>
             <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-download" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
               <path fill-rule="evenodd" d="M.5 9.9a.5.5 0 0 1 .5.5v2.5a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1v-2.5a.5.5 0 0 1 1 0v2.5a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2v-2.5a.5.5 0 0 1 .5-.5z"/>
               <path fill-rule="evenodd" d="M7.646 11.854a.5.5 0 0 0 .708 0l3-3a.5.5 0 0 0-.708-.708L8.5 10.293V1.5a.5.5 0 0 0-1 0v8.793L5.354 8.146a.5.5 0 1 0-.708.708l3 3z"/>
             </svg>
           </a>
         </div>
-        <small class="text-muted pl-1">{{document.created | date}}</small>
+        <small class="text-muted pl-1">{{document.created | customDate:'shortDate'}}</small>
       </div>
 
     </div>

src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss

@@ -2,7 +2,7 @@
 
 .doc-img {
   object-fit: cover;
-  object-position: top;
+  object-position: top left;
   height: 200px;
   mix-blend-mode: multiply;
 }

src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts

@@ -14,14 +14,9 @@ export class DocumentCardSmallComponent implements OnInit {
 
   @Input()
   selected = false
-
-  setSelected(value: boolean) {
-    this.selected = value
-    this.selectedChange.emit(value)
-  }
-
+  
   @Output()
-  selectedChange = new EventEmitter<boolean>()
+  toggleSelected = new EventEmitter()
 
   @Input()
   document: PaperlessDocument

src-ui/src/app/components/document-list/document-list.component.html

@@ -63,12 +63,12 @@
   <div class="btn-group ml-2 flex-fill" ngbDropdown role="group">
     <button class="btn btn-sm btn-outline-primary dropdown-toggle flex-fill" ngbDropdownToggle i18n>Views</button>
     <div class="dropdown-menu shadow dropdown-menu-right" ngbDropdownMenu>
-      <ng-container *ngIf="!list.savedViewId">
+      <ng-container *ngIf="!list.activeSavedViewId">
         <button ngbDropdownItem *ngFor="let view of savedViewService.allViews" (click)="loadViewConfig(view)">{{view.name}}</button>
         <div class="dropdown-divider" *ngIf="savedViewService.allViews.length > 0"></div>
       </ng-container>
 
-      <button ngbDropdownItem (click)="saveViewConfig()" *ngIf="list.savedViewId" i18n>Save "{{list.savedViewTitle}}"</button>
+      <button ngbDropdownItem (click)="saveViewConfig()" *ngIf="list.activeSavedViewId" i18n>Save "{{list.activeSavedViewTitle}}"</button>
       <button ngbDropdownItem (click)="saveViewConfigAs()" i18n>Save as...</button>
     </div>
   </div>
@@ -86,11 +86,11 @@
     <span i18n *ngIf="list.selected.size == 0">{list.collectionSize, plural, =1 {One document} other {{{list.collectionSize || 0}} documents}}</span>&nbsp;<span i18n *ngIf="isFiltered">(filtered)</span>
   </p>
   <ngb-pagination [pageSize]="list.currentPageSize" [collectionSize]="list.collectionSize" [(page)]="list.currentPage" [maxSize]="5"
-  [rotate]="true" (pageChange)="list.reload()" aria-label="Default pagination"></ngb-pagination>
+  [rotate]="true" aria-label="Default pagination"></ngb-pagination>
 </div>
 
 <div *ngIf="displayMode == 'largeCards'">
-  <app-document-card-large [selected]="list.isSelected(d)" (selectedChange)="list.setSelected(d, $event)"   *ngFor="let d of list.documents; trackBy: trackByDocumentId" [document]="d" [details]="d.content" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)">
+  <app-document-card-large [selected]="list.isSelected(d)" (toggleSelected)="toggleSelected(d, $event)" *ngFor="let d of list.documents; trackBy: trackByDocumentId" [document]="d" [details]="d.content" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)">
   </app-document-card-large>
 </div>
 
@@ -135,10 +135,10 @@
       i18n>Added</th>
   </thead>
   <tbody>
-    <tr *ngFor="let d of list.documents; trackBy: trackByDocumentId" [ngClass]="list.isSelected(d) ? 'table-row-selected' : ''">
+    <tr *ngFor="let d of list.documents; trackBy: trackByDocumentId" (click)="toggleSelected(d, $event)" [ngClass]="list.isSelected(d) ? 'table-row-selected' : ''">
       <td>
         <div class="custom-control custom-checkbox">
-          <input type="checkbox" class="custom-control-input" id="docCheck{{d.id}}" [checked]="list.isSelected(d)" (change)="list.setSelected(d, $event.target.checked)">
+          <input type="checkbox" class="custom-control-input" id="docCheck{{d.id}}" [checked]="list.isSelected(d)" (click)="toggleSelected(d, $event)">
           <label class="custom-control-label" for="docCheck{{d.id}}"></label>
         </div>
       </td>
@@ -147,28 +147,28 @@
       </td>
       <td class="d-none d-md-table-cell">
         <ng-container *ngIf="d.correspondent">
-          <a [routerLink]="" (click)="clickCorrespondent(d.correspondent)" title="Filter by correspondent">{{(d.correspondent$ | async)?.name}}</a>
+          <a [routerLink]="" (click)="clickCorrespondent(d.correspondent);$event.stopPropagation()" title="Filter by correspondent">{{(d.correspondent$ | async)?.name}}</a>
         </ng-container>
       </td>
       <td>
         <a routerLink="/documents/{{d.id}}" title="Edit document" style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
-        <app-tag [tag]="t" *ngFor="let t of d.tags$ | async" class="ml-1" clickable="true" linkTitle="Filter by tag" (click)="clickTag(t.id)"></app-tag>
+        <app-tag [tag]="t" *ngFor="let t of d.tags$ | async" class="ml-1" clickable="true" linkTitle="Filter by tag" (click)="clickTag(t.id);$event.stopPropagation()"></app-tag>
       </td>
       <td class="d-none d-xl-table-cell">
         <ng-container *ngIf="d.document_type">
-          <a [routerLink]="" (click)="clickDocumentType(d.document_type)" title="Filter by document type">{{(d.document_type$ | async)?.name}}</a>
+          <a [routerLink]="" (click)="clickDocumentType(d.document_type);$event.stopPropagation()" title="Filter by document type">{{(d.document_type$ | async)?.name}}</a>
         </ng-container>
       </td>
       <td>
-        {{d.created | date}}
+        {{d.created | customDate}}
       </td>
       <td class="d-none d-xl-table-cell">
-        {{d.added | date}}
+        {{d.added | customDate}}
       </td>
     </tr>
   </tbody>
 </table>
 
 <div class="m-n2 row row-cols-paperless-cards" *ngIf="displayMode == 'smallCards'">
-  <app-document-card-small [selected]="list.isSelected(d)" (selectedChange)="list.setSelected(d, $event)"  [document]="d" *ngFor="let d of list.documents; trackBy: trackByDocumentId" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)"></app-document-card-small>
+  <app-document-card-small [selected]="list.isSelected(d)" (toggleSelected)="toggleSelected(d, $event)"  [document]="d" *ngFor="let d of list.documents; trackBy: trackByDocumentId" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)"></app-document-card-small>
 </div>

src-ui/src/app/components/document-list/document-list.component.scss

@@ -1,5 +1,9 @@
 @import "/src/theme";
 
+tr {
+  user-select: none;
+}
+
 .table-row-selected {
   background-color: $primaryFaded;
 }

src-ui/src/app/components/document-list/document-list.component.ts

@@ -1,13 +1,15 @@
-import { AfterViewInit, Component, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
+import { Component, OnDestroy, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
+import { Subscription } from 'rxjs';
 import { PaperlessDocument } from 'src/app/data/paperless-document';
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view';
 import { SortableDirective, SortEvent } from 'src/app/directives/sortable.directive';
+import { ConsumerStatusService } from 'src/app/services/consumer-status.service';
 import { DocumentListViewService } from 'src/app/services/document-list-view.service';
 import { DOCUMENT_SORT_FIELDS } from 'src/app/services/rest/document.service';
 import { SavedViewService } from 'src/app/services/rest/saved-view.service';
-import { Toast, ToastService } from 'src/app/services/toast.service';
+import { ToastService } from 'src/app/services/toast.service';
 import { FilterEditorComponent } from './filter-editor/filter-editor.component';
 import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-view-config-dialog.component';
 
@@ -16,7 +18,7 @@ import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-vi
   templateUrl: './document-list.component.html',
   styleUrls: ['./document-list.component.scss']
 })
-export class DocumentListComponent implements OnInit {
+export class DocumentListComponent implements OnInit, OnDestroy {
 
   constructor(
     public list: DocumentListViewService,
@@ -24,7 +26,9 @@ export class DocumentListComponent implements OnInit {
     public route: ActivatedRoute,
     private router: Router,
     private toastService: ToastService,
-    private modalService: NgbModal) { }
+    private modalService: NgbModal,
+    private consumerStatusService: ConsumerStatusService
+  ) { }
 
   @ViewChild("filterEditor")
   private filterEditor: FilterEditorComponent
@@ -35,12 +39,14 @@ export class DocumentListComponent implements OnInit {
 
   filterRulesModified: boolean = false
 
+  private consumptionFinishedSubscription: Subscription
+
   get isFiltered() {
     return this.list.filterRules?.length > 0
   }
 
   getTitle() {
-    return this.list.savedViewTitle || $localize`Documents`
+    return this.list.activeSavedViewTitle || $localize`Documents`
   }
 
   getSortFields() {
@@ -63,37 +69,52 @@ export class DocumentListComponent implements OnInit {
     if (localStorage.getItem('document-list:displayMode') != null) {
       this.displayMode = localStorage.getItem('document-list:displayMode')
     }
+    this.consumptionFinishedSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(() => {
+      this.list.reload()
+    })
     this.route.paramMap.subscribe(params => {
-      this.list.clear()
       if (params.has('id')) {
         this.savedViewService.getCached(+params.get('id')).subscribe(view => {
           if (!view) {
             this.router.navigate(["404"])
             return
           }
-          this.list.savedView = view
+          this.list.activateSavedView(view)
           this.list.reload()
           this.rulesChanged()
         })
       } else {
-        this.list.savedView = null
+        this.list.activateSavedView(null)
         this.list.reload()
         this.rulesChanged()
       }
     })
   }
 
+  ngOnDestroy() {
+    if (this.consumptionFinishedSubscription) {
+      this.consumptionFinishedSubscription.unsubscribe()
+    }
+  }
+
   loadViewConfig(view: PaperlessSavedView) {
-    this.list.load(view)
+    this.list.loadSavedView(view)
     this.list.reload()
     this.rulesChanged()
   }
 
   saveViewConfig() {
-    this.savedViewService.update(this.list.savedView).subscribe(result => {
-      this.toastService.showInfo($localize`View "${this.list.savedView.name}" saved successfully.`)
-    })
-
+    if (this.list.activeSavedViewId != null) {
+      let savedView: PaperlessSavedView = {
+        id: this.list.activeSavedViewId,
+        filter_rules: this.list.filterRules,
+        sort_field: this.list.sortField,
+        sort_reverse: this.list.sortReverse
+      }
+      this.savedViewService.patch(savedView).subscribe(result => {
+        this.toastService.showInfo($localize`View "${this.list.activeSavedViewTitle}" saved successfully.`)
+      })
+    }
   }
 
   saveViewConfigAs() {
@@ -101,7 +122,7 @@ export class DocumentListComponent implements OnInit {
     modal.componentInstance.defaultName = this.filterEditor.generateFilterName()
     modal.componentInstance.saveClicked.subscribe(formValue => {
       modal.componentInstance.buttonsEnabled = false
-      let savedView = {
+      let savedView: PaperlessSavedView = {
         name: formValue.name,
         show_on_dashboard: formValue.showOnDashboard,
         show_in_sidebar: formValue.showInSideBar,
@@ -122,8 +143,8 @@ export class DocumentListComponent implements OnInit {
 
   resetFilters(): void {
     this.filterRulesModified = false
-    if (this.list.savedViewId) {
-      this.savedViewService.getCached(this.list.savedViewId).subscribe(viewUntouched => {
+    if (this.list.activeSavedViewId) {
+      this.savedViewService.getCached(this.list.activeSavedViewId).subscribe(viewUntouched => {
         this.list.filterRules = viewUntouched.filter_rules
         this.list.reload()
       })
@@ -135,11 +156,11 @@ export class DocumentListComponent implements OnInit {
 
   rulesChanged() {
     let modified = false
-    if (this.list.savedView == null) {
+    if (this.list.activeSavedViewId == null) {
       modified = this.list.filterRules.length > 0 // documents list is modified if it has any filters
     } else {
       // compare savedView current filters vs original
-      this.savedViewService.getCached(this.list.savedViewId).subscribe(view => {
+      this.savedViewService.getCached(this.list.activeSavedViewId).subscribe(view => {
         let filterRulesInitial = view.filter_rules
 
         if (this.list.filterRules.length !== filterRulesInitial.length) modified = true
@@ -160,6 +181,11 @@ export class DocumentListComponent implements OnInit {
     this.filterRulesModified = modified
   }
 
+  toggleSelected(document: PaperlessDocument, event: MouseEvent): void {
+    if (!event.shiftKey) this.list.toggleSelected(document)
+    else this.list.selectRangeTo(document)
+  }
+
   clickTag(tagID: number) {
     this.list.selectNone()
     setTimeout(() => {

src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts

@@ -46,6 +46,8 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
             return $localize`Without any tag`
           }
 
+        case FILTER_TITLE:
+          return $localize`Title: ${rule.value}`
       }
     }
 
@@ -117,7 +119,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
     })
   }
 
-  get filterRules() {
+  get filterRules(): FilterRule[] {
     let filterRules: FilterRule[] = []
     if (this._titleFilter) {
       filterRules.push({rule_type: FILTER_TITLE, value: this._titleFilter})

src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.html

@@ -1,7 +1,7 @@
 <form [formGroup]="saveViewConfigForm" (ngSubmit)="save()">
   <div class="modal-header">
     <h4 class="modal-title" id="modal-basic-title" i18n>Save current view</h4>
-    <button type="button" class="close" aria-label="Close" (click)="cancel()">
+    <button type="button" [disabled]="!closeEnabled" class="close" aria-label="Close" (click)="cancel()">
       <span aria-hidden="true">&times;</span>
     </button>
   </div>

src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.ts

@@ -20,6 +20,8 @@ export class SaveViewConfigDialogComponent implements OnInit {
   @Input()
   buttonsEnabled = true
 
+  closeEnabled = false
+
   _defaultName = ""
 
   get defaultName() {
@@ -31,7 +33,7 @@ export class SaveViewConfigDialogComponent implements OnInit {
     this._defaultName = value
     this.saveViewConfigForm.patchValue({name: value})
   }
-  
+
   saveViewConfigForm = new FormGroup({
     name: new FormControl(''),
     showInSideBar: new FormControl(false),
@@ -39,6 +41,10 @@ export class SaveViewConfigDialogComponent implements OnInit {
   })
 
   ngOnInit(): void {
+    // wait to enable close button so it doesnt steal focus from input since its the first clickable element in the DOM
+    setTimeout(() => {
+      this.closeEnabled = true
+    });
   }
 
   save() {

src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html

@@ -1,12 +1,11 @@
 <form [formGroup]="objectForm" (ngSubmit)="save()">
   <div class="modal-header">
     <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
-    <button type="button" class="close" aria-label="Close" (click)="cancel()">
+    <button type="button" [disabled]="!closeEnabled" class="close" aria-label="Close" (click)="cancel()">
       <span aria-hidden="true">&times;</span>
     </button>
   </div>
   <div class="modal-body">
-
     <app-input-text i18n-title title="Name" formControlName="name" [error]="error?.name"></app-input-text>
     <app-input-select i18n-title title="Matching algorithm" [items]="getMatchingAlgorithms()" formControlName="matching_algorithm"></app-input-select>
     <app-input-text *ngIf="patternRequired" i18n-title title="Matching pattern" formControlName="match"></app-input-text>

src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html

@@ -2,8 +2,15 @@
   <button type="button" class="btn btn-sm btn-outline-primary" (click)="openCreateDialog()" i18n>Create</button>
 </app-page-header>
 
-<div class="row m-0 justify-content-end">
-  <ngb-pagination [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
+<div class="row">
+  <div class="col-md mb-2 mb-xl-0">
+    <div class="form-inline d-flex align-items-center">
+      <label class="text-muted mr-2 mb-0" i18n>Filter by:</label>
+      <input class="form-control form-control-sm flex-fill w-auto" type="text" [(ngModel)]="nameFilter" placeholder="Name" i18n-placeholder>
+    </div>
+  </div>
+
+  <ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
 </div>
 
 <table class="table table-striped border shadow-sm">
@@ -21,7 +28,7 @@
       <td scope="row">{{ correspondent.name }}</td>
       <td scope="row">{{ getMatching(correspondent) }}</td>
       <td scope="row">{{ correspondent.document_count }}</td>
-      <td scope="row">{{ correspondent.last_correspondence | date }}</td>
+      <td scope="row">{{ correspondent.last_correspondence | customDate }}</td>
         <td scope="row">
           <div class="btn-group">
             <button class="btn btn-sm btn-outline-secondary" (click)="filterDocuments(correspondent)">

src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html

@@ -1,7 +1,7 @@
 <form [formGroup]="objectForm" (ngSubmit)="save()">
     <div class="modal-header">
       <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
-      <button type="button" class="close" aria-label="Close" (click)="cancel()">
+      <button type="button" [disabled]="!closeEnabled" class="close" aria-label="Close" (click)="cancel()">
         <span aria-hidden="true">&times;</span>
       </button>
     </div>

src-ui/src/app/components/manage/document-type-list/document-type-list.component.html

@@ -2,9 +2,15 @@
   <button type="button" class="btn btn-sm btn-outline-primary" (click)="openCreateDialog()" i18n>Create</button>
 </app-page-header>
 
-<div class="row m-0 justify-content-end">
-  <ngb-pagination [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()"
-  aria-label="Default pagination"></ngb-pagination>
+<div class="row">
+  <div class="col-md mb-2 mb-xl-0">
+    <div class="form-inline d-flex align-items-center">
+      <label class="text-muted mr-2 mb-0" i18n>Filter by:</label>
+      <input class="form-control form-control-sm flex-fill w-auto" type="text" [(ngModel)]="nameFilter" placeholder="Name" i18n-placeholder>
+    </div>
+  </div>
+
+  <ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
 </div>
 
 <table class="table table-striped border shadow-sm">

src-ui/src/app/components/manage/generic-list/generic-list.component.ts

@@ -1,17 +1,19 @@
-import { Directive, OnInit, QueryList, ViewChildren } from '@angular/core';
+import { Directive, OnDestroy, OnInit, QueryList, ViewChildren } from '@angular/core';
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
+import { Subject, Subscription } from 'rxjs';
+import { debounceTime, distinctUntilChanged } from 'rxjs/operators';
 import { MatchingModel, MATCHING_ALGORITHMS, MATCH_AUTO } from 'src/app/data/matching-model';
 import { ObjectWithId } from 'src/app/data/object-with-id';
 import { SortableDirective, SortEvent } from 'src/app/directives/sortable.directive';
-import { AbstractPaperlessService } from 'src/app/services/rest/abstract-paperless-service';
+import { AbstractNameFilterService } from 'src/app/services/rest/abstract-name-filter-service';
 import { ToastService } from 'src/app/services/toast.service';
 import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component';
 
 @Directive()
-export abstract class GenericListComponent<T extends ObjectWithId> implements OnInit {
+export abstract class GenericListComponent<T extends ObjectWithId> implements OnInit, OnDestroy {
 
   constructor(
-    private service: AbstractPaperlessService<T>,
+    private service: AbstractNameFilterService<T>,
     private modalService: NgbModal,
     private editDialogComponent: any,
     private toastService: ToastService) {
@@ -28,6 +30,10 @@ export abstract class GenericListComponent<T extends ObjectWithId> implements On
   public sortField: string
   public sortReverse: boolean
 
+  private nameFilterDebounce: Subject<string>
+  private subscription: Subscription
+  private _nameFilter: string
+
   getMatching(o: MatchingModel) {
     if (o.matching_algorithm == MATCH_AUTO) {
       return $localize`Automatic`
@@ -44,12 +50,27 @@ export abstract class GenericListComponent<T extends ObjectWithId> implements On
     this.reloadData()
   }
 
+
   ngOnInit(): void {
     this.reloadData()
+
+    this.nameFilterDebounce = new Subject<string>()
+
+    this.subscription = this.nameFilterDebounce.pipe(
+      debounceTime(400),
+      distinctUntilChanged()
+    ).subscribe(title => {
+      this._nameFilter = title
+      this.reloadData()
+    })
+  }
+
+  ngOnDestroy() {
+    this.subscription.unsubscribe()
   }
 
   reloadData() {
-    this.service.list(this.page, null, this.sortField, this.sortReverse).subscribe(c => {
+    this.service.listFiltered(this.page, null, this.sortField, this.sortReverse, this._nameFilter).subscribe(c => {
       this.data = c.results
       this.collectionSize = c.count
     });
@@ -95,4 +116,12 @@ export abstract class GenericListComponent<T extends ObjectWithId> implements On
     }
     )
   }
+
+  get nameFilter() {
+    return this._nameFilter
+  }
+
+  set nameFilter(nameFilter: string) {
+    this.nameFilterDebounce.next(nameFilter)
+  }
 }

src-ui/src/app/components/manage/logs/logs.component.html

@@ -1,27 +1,18 @@
 <app-page-header title="Logs" i18n-title>
 
-  <div ngbDropdown class="btn-group">
-    <button class="btn btn-outline-primary btn-sm" id="dropdownBasic1" ngbDropdownToggle>
-      <svg class="toolbaricon" fill="currentColor">
-        <use xlink:href="assets/bootstrap-icons.svg#funnel" />
-      </svg>&nbsp;<ng-container i18n>Filter</ng-container>
-
-
-    </button>
-    <div ngbDropdownMenu aria-labelledby="dropdownBasic1">
-      <button *ngFor="let f of getLevels()" ngbDropdownItem (click)="setLevel(f.id)"
-        [class.active]="level == f.id">{{f.name}}</button>
-    </div>
-  </div>
-
 </app-page-header>
 
-<div class="bg-dark p-3 mb-3 text-light text-monospace" infiniteScroll (scrolled)="onScroll()">
+
+<ul ngbNav #nav="ngbNav" [(activeId)]="activeLog" (activeIdChange)="reloadLogs()" class="nav-tabs">
+  <li *ngFor="let logFile of logFiles" [ngbNavItem]="logFile">
+    <a ngbNavLink>{{logFile}}.log</a>
+  </li>
+</ul>
+
+<div [ngbNavOutlet]="nav" class="mt-2"></div>
+
+<div class="bg-dark p-3 mb-3 text-light text-monospace log-container">
   <p
-    class="m-0 p-0 log-entry-{{log.level}}"
-    *ngFor="let log of logs">
-      {{log.created | date:'short'}}
-      {{getLevelText(log.level)}}
-      {{log.message}}
-  </p>
+    class="m-0 p-0 log-entry-{{getLogLevel(log)}}"
+    *ngFor="let log of logs" style="white-space: pre;">{{log}}</p>
 </div>

src-ui/src/app/components/manage/logs/logs.component.scss

@@ -13,4 +13,12 @@
 .log-entry-50 {
   color: lightcoral !important;
   font-weight: bold;
+}
+
+.log-container {
+
+  overflow: scroll;
+
+  height: calc(100vh - 190px);
+  top: 70px;
 }

src-ui/src/app/components/manage/logs/logs.component.ts

@@ -1,5 +1,4 @@
-import { Component, OnInit } from '@angular/core';
-import { LOG_LEVELS, LOG_LEVEL_INFO, PaperlessLog } from 'src/app/data/paperless-log';
+import { Component, ElementRef, OnInit, ViewChild } from '@angular/core';
 import { LogService } from 'src/app/services/rest/log.service';
 
 @Component({
@@ -11,38 +10,42 @@ export class LogsComponent implements OnInit {
 
   constructor(private logService: LogService) { }
 
-  logs: PaperlessLog[] = []
-  level: number = LOG_LEVEL_INFO
+  logs: string[] = []
+
+  logFiles: string[] = []
+
+  activeLog: string
 
   ngOnInit(): void {
-    this.reload()
-  }
-
-  reload() {
-    this.logService.list(1, 50, 'created', true, {'level__gte': this.level}).subscribe(result => this.logs = result.results)
-  }
-
-  getLevelText(level: number) {
-    return LOG_LEVELS.find(l => l.id == level)?.name
-  }
-
-  onScroll() {
-    let lastCreated = null
-    if (this.logs.length > 0) {
-      lastCreated = new Date(this.logs[this.logs.length-1].created).toISOString()
-    }
-    this.logService.list(1, 25, 'created', true, {'created__lt': lastCreated, 'level__gte': this.level}).subscribe(result => {
-      this.logs.push(...result.results)
+    this.logService.list().subscribe(result => {
+      this.logFiles = result
+      if (this.logFiles.length > 0) {
+        this.activeLog = this.logFiles[0]
+        this.reloadLogs()
+      }
     })
   }
 
-  getLevels() {
-    return LOG_LEVELS
+  reloadLogs() {
+    this.logService.get(this.activeLog).subscribe(result => {
+      this.logs = result
+    }, error => {
+      this.logs = []
+    })
   }
 
-  setLevel(id) {
-    this.level = id
-    this.reload()
+  getLogLevel(log: string) {
+    if (log.indexOf("[DEBUG]") != -1) {
+      return 10
+    } else if (log.indexOf("[WARNING]") != -1) {
+      return 30
+    } else if (log.indexOf("[ERROR]") != -1) {
+      return 40
+    } else if (log.indexOf("[CRITICAL]") != -1) {
+      return 50
+    } else {
+      return 20
+    }
   }
 
 }

src-ui/src/app/components/manage/settings/settings.component.html

@@ -12,6 +12,56 @@
 
         <h4 i18n>Appearance</h4>
 
+        <div class="form-row form-group">
+          <div class="col-md-3 col-form-label">
+            <span i18n>Display language</span>
+          </div>
+          <div class="col">
+
+            <select class="form-control" formControlName="displayLanguage">
+              <option *ngFor="let lang of displayLanguageOptions" [ngValue]="lang.code">{{lang.name}}<span *ngIf="lang.code && currentLocale != 'en-US'"> - {{lang.englishName}}</span></option>
+            </select>
+
+            <small class="form-text text-muted" i18n>You need to reload the page after applying a new language.</small>
+
+          </div>
+        </div>
+
+        <div class="form-row form-group">
+          <div class="col-md-3 col-form-label">
+            <span i18n>Date display</span>
+          </div>
+          <div class="col">
+
+            <select class="form-control" formControlName="dateLocale">
+              <option *ngFor="let lang of dateLocaleOptions" [ngValue]="lang.code">{{lang.name}}<span *ngIf="lang.code"> - {{today | customDate:'shortDate':null:lang.code}}</span></option>
+            </select>
+
+          </div>
+        </div>
+
+        <div class="form-row form-group">
+          <div class="col-md-3 col-form-label">
+            <span i18n>Date format</span>
+          </div>
+          <div class="col">
+
+            <div class="custom-control custom-radio">
+              <input type="radio" id="dateFormatShort" name="dateFormat" class="custom-control-input" formControlName="dateFormat" value="shortDate">
+              <label class="custom-control-label" for="dateFormatShort" i18n>Short: {{today | customDate:'shortDate':null:computedDateLocale}}</label>
+            </div>
+            <div class="custom-control custom-radio">
+              <input type="radio" id="dateFormatMedium" name="dateFormat" class="custom-control-input" formControlName="dateFormat" value="mediumDate">
+              <label class="custom-control-label" for="dateFormatMedium" i18n>Medium: {{today | customDate:'mediumDate':null:computedDateLocale}}</label>
+            </div>
+            <div class="custom-control custom-radio">
+              <input type="radio" id="dateFormatLong" name="dateFormat" class="custom-control-input" formControlName="dateFormat" value="longDate">
+              <label class="custom-control-label" for="dateFormatLong" i18n>Long: {{today | customDate:'longDate':null:computedDateLocale}}</label>
+            </div>
+
+          </div>
+        </div>
+
         <div class="form-row form-group">
           <div class="col-md-3 col-form-label">
             <span i18n>Items per page</span>
@@ -60,7 +110,26 @@
 
       </ng-template>
     </li>
+    
     <li [ngbNavItem]="2">
+      <a ngbNavLink i18n>Notifications</a>
+      <ng-template ngbNavContent>
+        
+        <h4 i18n>Document processing</h4>
+
+        <div class="form-row form-group">
+          <div class="offset-md-3 col">
+            <app-input-check i18n-title title="Show notifications when new documents are detected" formControlName="notificationsConsumerNewDocument"></app-input-check>
+            <app-input-check i18n-title title="Show notifications when document processing completes successfully" formControlName="notificationsConsumerSuccess"></app-input-check>
+            <app-input-check i18n-title title="Show notifications when document processing fails" formControlName="notificationsConsumerFailed"></app-input-check>
+            <app-input-check i18n-title title="Suppress notifications on dashboard" formControlName="notificationsConsumerSuppressOnDashboard" i18n-hint hint="This will suppress all messages about document processing status on the dashboard."></app-input-check>
+          </div>
+        </div>
+
+      </ng-template>
+    </li>
+
+    <li [ngbNavItem]="3">
       <a ngbNavLink i18n>Saved views</a>
       <ng-template ngbNavContent>
 
@@ -98,7 +167,7 @@
     </li>
   </ul>
 
-  <div [ngbNavOutlet]="nav" class="border-left border-right border-bottom p-3 mb-3 shadow"></div>
+  <div [ngbNavOutlet]="nav" class="border-left border-right border-bottom p-3 mb-3 shadow-sm"></div>
 
   <button type="submit" class="btn btn-primary" i18n>Save</button>
 </form>

src-ui/src/app/components/manage/settings/settings.component.ts

@@ -1,9 +1,9 @@
-import { Component, OnInit, Renderer2  } from '@angular/core';
+import { Component, Inject, LOCALE_ID, OnInit, Renderer2  } from '@angular/core';
 import { FormControl, FormGroup } from '@angular/forms';
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view';
 import { DocumentListViewService } from 'src/app/services/document-list-view.service';
 import { SavedViewService } from 'src/app/services/rest/saved-view.service';
-import { SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
+import { LanguageOption, SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
 import { ToastService } from 'src/app/services/toast.service';
 
 @Component({
@@ -22,16 +22,28 @@ export class SettingsComponent implements OnInit {
     'darkModeUseSystem': new FormControl(this.settings.get(SETTINGS_KEYS.DARK_MODE_USE_SYSTEM)),
     'darkModeEnabled': new FormControl(this.settings.get(SETTINGS_KEYS.DARK_MODE_ENABLED)),
     'useNativePdfViewer': new FormControl(this.settings.get(SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER)),
-    'savedViews': this.savedViewGroup
+    'savedViews': this.savedViewGroup,
+    'displayLanguage': new FormControl(this.settings.getLanguage()),
+    'dateLocale': new FormControl(this.settings.get(SETTINGS_KEYS.DATE_LOCALE)),
+    'dateFormat': new FormControl(this.settings.get(SETTINGS_KEYS.DATE_FORMAT)),
+    'notificationsConsumerNewDocument': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT)),
+    'notificationsConsumerSuccess': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS)),
+    'notificationsConsumerFailed': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED)),
+    'notificationsConsumerSuppressOnDashboard': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD)),
   })
 
   savedViews: PaperlessSavedView[]
 
+  get computedDateLocale(): string {
+    return this.settingsForm.value.dateLocale || this.settingsForm.value.displayLanguage || this.currentLocale
+  }
+
   constructor(
     public savedViewService: SavedViewService,
     private documentListViewService: DocumentListViewService,
     private toastService: ToastService,
-    private settings: SettingsService
+    private settings: SettingsService,
+    @Inject(LOCALE_ID) public currentLocale: string
   ) { }
 
   ngOnInit() {
@@ -63,11 +75,33 @@ export class SettingsComponent implements OnInit {
     this.settings.set(SETTINGS_KEYS.DARK_MODE_USE_SYSTEM, this.settingsForm.value.darkModeUseSystem)
     this.settings.set(SETTINGS_KEYS.DARK_MODE_ENABLED, (this.settingsForm.value.darkModeEnabled == true).toString())
     this.settings.set(SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER, this.settingsForm.value.useNativePdfViewer)
+    this.settings.set(SETTINGS_KEYS.DATE_LOCALE, this.settingsForm.value.dateLocale)
+    this.settings.set(SETTINGS_KEYS.DATE_FORMAT, this.settingsForm.value.dateFormat)
+    this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT, this.settingsForm.value.notificationsConsumerNewDocument)
+    this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS, this.settingsForm.value.notificationsConsumerSuccess)
+    this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED, this.settingsForm.value.notificationsConsumerFailed)
+    this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD, this.settingsForm.value.notificationsConsumerSuppressOnDashboard)
+    this.settings.setLanguage(this.settingsForm.value.displayLanguage)
     this.documentListViewService.updatePageSize()
     this.settings.updateDarkModeSettings()
     this.toastService.showInfo($localize`Settings saved successfully.`)
   }
 
+  get displayLanguageOptions(): LanguageOption[] {
+    return [{code: "", name: $localize`Use system language`}].concat(this.settings.getLanguageOptions())
+  }
+
+  get dateLocaleOptions(): LanguageOption[] {
+    return [
+      {code: "", name: $localize`Use date format of display language`},
+      {code: "iso-8601", name: $localize`ISO 8601`}
+    ].concat(this.settings.getLanguageOptions())
+  }
+
+  get today() {
+    return new Date()
+  }
+
   saveSettings() {
     let x = []
     for (let id in this.savedViewGroup.value) {

src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html

@@ -1,7 +1,7 @@
   <form [formGroup]="objectForm" (ngSubmit)="save()">
     <div class="modal-header">
       <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
-      <button type="button" class="close" aria-label="Close" (click)="cancel()">
+      <button type="button" [disabled]="!closeEnabled" class="close" aria-label="Close" (click)="cancel()">
         <span aria-hidden="true">&times;</span>
       </button>
     </div>

src-ui/src/app/components/manage/tag-list/tag-list.component.html

@@ -2,9 +2,15 @@
   <button type="button" class="btn btn-sm btn-outline-primary" (click)="openCreateDialog()" i18n>Create</button>
 </app-page-header>
 
-<div class="row m-0 justify-content-end">
-  <ngb-pagination [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()"
-    aria-label="Default pagination"></ngb-pagination>
+<div class="row">
+  <div class="col-md mb-2 mb-xl-0">
+    <div class="form-inline d-flex align-items-center">
+      <label class="text-muted mr-2 mb-0" i18n>Filter by:</label>
+      <input class="form-control form-control-sm flex-fill w-auto" type="text" [(ngModel)]="nameFilter" placeholder="Name" i18n-placeholder>
+    </div>
+  </div>
+
+  <ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
 </div>
 
 <table class="table table-striped border shadow-sm">

src-ui/src/app/data/paperless-document-suggestions.ts

@@ -0,0 +1,9 @@
+export interface PaperlessDocumentSuggestions {
+
+  tags?: number[]
+
+  correspondents?: number[]
+
+  document_types?: number[]
+
+}

src-ui/src/app/data/paperless-log.ts

@@ -1,27 +0,0 @@
-export const LOG_LEVEL_DEBUG = 10
-export const LOG_LEVEL_INFO = 20
-export const LOG_LEVEL_WARNING = 30
-export const LOG_LEVEL_ERROR = 40
-export const LOG_LEVEL_CRITICAL = 50
-
-export const LOG_LEVELS = [
-  {id: LOG_LEVEL_DEBUG, name: "DEBUG"},
-  {id: LOG_LEVEL_INFO, name: "INFO"},
-  {id: LOG_LEVEL_WARNING, name: "WARNING"},
-  {id: LOG_LEVEL_ERROR, name: "ERROR"},
-  {id: LOG_LEVEL_CRITICAL, name: "CRITICAL"}
-]
-
-export interface PaperlessLog {
-
-  id?: number
-
-  group?: string
-
-  message?: string
-
-  created?: Date
-
-  level?: number
-
-}

src-ui/src/app/data/websocket-consumer-status-message.ts

@@ -0,0 +1,11 @@
+export interface WebsocketConsumerStatusMessage {
+
+  filename?: string
+  task_id?: string
+  current_progress?: number
+  max_progress?: number
+  status?: string
+  message?: string
+  document_id: number
+
+}

src-ui/src/app/pipes/custom-date.pipe.spec.ts

@@ -0,0 +1,8 @@
+import { CustomDatePipe } from './custom-date.pipe';
+
+describe('CustomDatePipe', () => {
+  it('create an instance', () => {
+    const pipe = new CustomDatePipe();
+    expect(pipe).toBeTruthy();
+  });
+});

src-ui/src/app/pipes/custom-date.pipe.ts

@@ -0,0 +1,30 @@
+import { DatePipe } from '@angular/common';
+import { Inject, LOCALE_ID, Pipe, PipeTransform } from '@angular/core';
+import { SettingsService, SETTINGS_KEYS } from '../services/settings.service';
+
+const FORMAT_TO_ISO_FORMAT = {
+  "longDate": "y-MM-dd",
+  "mediumDate": "yy-MM-dd",
+  "shortDate": "yy-MM-dd"
+}
+
+@Pipe({
+  name: 'customDate'
+})
+export class CustomDatePipe extends DatePipe implements PipeTransform {
+
+  constructor(@Inject(LOCALE_ID) locale: string, private settings: SettingsService) {
+    super(locale)
+  }
+
+  transform(value: any, format?: string, timezone?: string, locale?: string): string | null {
+    let l = locale || this.settings.get(SETTINGS_KEYS.DATE_LOCALE)
+    let f = format || this.settings.get(SETTINGS_KEYS.DATE_FORMAT)
+    if (l == "iso-8601") {
+      return super.transform(value, FORMAT_TO_ISO_FORMAT[f], timezone)
+    } else {
+      return super.transform(value, format || this.settings.get(SETTINGS_KEYS.DATE_FORMAT), timezone, locale)
+    }
+  }
+
+}

src-ui/src/app/services/consumer-status.service.spec.ts

@@ -0,0 +1,16 @@
+import { TestBed } from '@angular/core/testing';
+
+import { ConsumerStatusService } from './consumer-status.service';
+
+describe('ConsumerStatusService', () => {
+  let service: ConsumerStatusService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({});
+    service = TestBed.inject(ConsumerStatusService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});

src-ui/src/app/services/consumer-status.service.ts

@@ -0,0 +1,200 @@
+import { Injectable } from '@angular/core';
+import { Subject } from 'rxjs';
+import { environment } from 'src/environments/environment';
+import { WebsocketConsumerStatusMessage } from '../data/websocket-consumer-status-message';
+
+export enum FileStatusPhase {
+  STARTED = 0,
+  UPLOADING = 1,
+  PROCESSING = 2,
+  SUCCESS = 3,
+  FAILED = 4
+}
+
+export const FILE_STATUS_MESSAGES = {
+  "document_already_exists": $localize`Document already exists.`,
+  "file_not_found": $localize`File not found.`,
+  "pre_consume_script_not_found": $localize`:Pre-Consume is a term that appears like that in the documentation as well and does not need a specific translation:Pre-consume script does not exist.`,
+  "pre_consume_script_error": $localize`:Pre-Consume is a term that appears like that in the documentation as well and does not need a specific translation:Error while executing pre-consume script.`,
+  "post_consume_script_not_found": $localize`:Post-Consume is a term that appears like that in the documentation as well and does not need a specific translation:Post-consume script does not exist.`,
+  "post_consume_script_error": $localize`:Post-Consume is a term that appears like that in the documentation as well and does not need a specific translation:Error while executing post-consume script.`,
+  "new_file": $localize`Received new file.`,
+  "unsupported_type": $localize`File type not supported.`,
+  "parsing_document": $localize`Processing document...`,
+  "generating_thumbnail": $localize`Generating thumbnail...`,
+  "parse_date": $localize`Retrieving date from document...`,
+  "save_document": $localize`Saving document...`,
+  "finished": $localize`Finished.`
+}
+
+export class FileStatus {
+
+  filename: string
+
+  taskId: string
+
+  phase: FileStatusPhase = FileStatusPhase.STARTED
+
+  currentPhaseProgress: number
+
+  currentPhaseMaxProgress: number
+
+  message: string
+
+  documentId: number
+
+  getProgress(): number {
+    switch (this.phase) {
+      case FileStatusPhase.STARTED:
+        return 0.0
+      case FileStatusPhase.UPLOADING:
+        return this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.2
+      case FileStatusPhase.PROCESSING:
+        return (this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.8) + 0.2
+      case FileStatusPhase.SUCCESS:
+      case FileStatusPhase.FAILED:
+        return 1.0
+    }
+  }
+
+  updateProgress(status: FileStatusPhase, currentProgress?: number, maxProgress?: number) {
+    if (status >= this.phase) {
+      this.phase = status
+      if (currentProgress != null) {
+        this.currentPhaseProgress = currentProgress
+      }
+      if (maxProgress != null) {
+        this.currentPhaseMaxProgress = maxProgress
+      }
+    }
+  }
+
+}
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ConsumerStatusService {
+
+  constructor() { }
+
+  private statusWebSocket: WebSocket
+
+  private consumerStatus: FileStatus[] = []
+
+  private documentDetectedSubject = new Subject<FileStatus>()
+  private documentConsumptionFinishedSubject = new Subject<FileStatus>()
+  private documentConsumptionFailedSubject = new Subject<FileStatus>()
+
+  private get(taskId: string, filename?: string) {
+    let status = this.consumerStatus.find(e => e.taskId == taskId) || this.consumerStatus.find(e => e.filename == filename && e.taskId == null)
+    let created = false
+    if (!status) {
+      status = new FileStatus()
+      this.consumerStatus.push(status)
+      created = true
+    }
+    status.taskId = taskId
+    status.filename = filename
+    return {'status': status, 'created': created}
+  }
+
+  newFileUpload(filename: string): FileStatus {
+    let status = new FileStatus()
+    status.filename = filename
+    this.consumerStatus.push(status)
+    return status
+  }
+
+  getConsumerStatus(phase?: FileStatusPhase) {
+    if (phase != null) {
+      return this.consumerStatus.filter(s => s.phase == phase)
+    } else {
+      return this.consumerStatus
+    }
+  }
+
+  getConsumerStatusNotCompleted() {
+    return this.consumerStatus.filter(s => s.phase < FileStatusPhase.SUCCESS)
+  }
+
+  getConsumerStatusCompleted() {
+    return this.consumerStatus.filter(s => s.phase == FileStatusPhase.FAILED || s.phase == FileStatusPhase.SUCCESS)
+  }
+
+  connect() {
+    this.disconnect()
+
+    this.statusWebSocket = new WebSocket(`${environment.webSocketProtocol}//${environment.webSocketHost}/ws/status/`);
+    this.statusWebSocket.onmessage = (ev) => {
+      let statusMessage: WebsocketConsumerStatusMessage = JSON.parse(ev['data'])
+
+      let statusMessageGet = this.get(statusMessage.task_id, statusMessage.filename)
+      let status = statusMessageGet.status
+      let created = statusMessageGet.created
+
+      status.updateProgress(FileStatusPhase.PROCESSING, statusMessage.current_progress, statusMessage.max_progress)
+      if (statusMessage.message && statusMessage.message in FILE_STATUS_MESSAGES) {
+        status.message = FILE_STATUS_MESSAGES[statusMessage.message]
+      } else if (statusMessage.message) {
+        status.message = statusMessage.message
+      }
+      status.documentId = statusMessage.document_id
+
+      if (created && statusMessage.status == 'STARTING') {
+        this.documentDetectedSubject.next(status)
+      }
+      if (statusMessage.status == "SUCCESS") {
+        status.phase = FileStatusPhase.SUCCESS
+        this.documentConsumptionFinishedSubject.next(status)
+      }
+      if (statusMessage.status == "FAILED") {
+        status.phase = FileStatusPhase.FAILED
+        this.documentConsumptionFailedSubject.next(status)
+      }
+    }
+  }
+
+  fail(status: FileStatus, message: string) {
+    status.message = message
+    status.phase = FileStatusPhase.FAILED
+    this.documentConsumptionFailedSubject.next(status)
+  }
+
+  disconnect() {
+    if (this.statusWebSocket) {
+      this.statusWebSocket.close()
+      this.statusWebSocket = null
+    }
+  }
+
+  dismiss(status: FileStatus) {
+    let index
+    if (status.taskId != null) {
+      index = this.consumerStatus.findIndex(s => s.taskId == status.taskId)
+    } else {
+      index = this.consumerStatus.findIndex(s => s.filename == status.filename)
+    }
+
+    if (index > -1) {
+      this.consumerStatus.splice(index, 1)
+    }
+  }
+
+  dismissCompleted() {
+    this.consumerStatus = this.consumerStatus.filter(status => status.phase != FileStatusPhase.SUCCESS)
+  }
+
+  onDocumentConsumptionFinished() {
+    return this.documentConsumptionFinishedSubject
+  }
+
+  onDocumentConsumptionFailed() {
+    return this.documentConsumptionFailedSubject
+  }
+
+  onDocumentDetected() {
+    return this.documentDetectedSubject
+  }
+
+}

src-ui/src/app/services/document-list-view.service.ts

@@ -8,6 +8,23 @@ import { DOCUMENT_LIST_SERVICE } from '../data/storage-keys';
 import { DocumentService } from './rest/document.service';
 import { SettingsService, SETTINGS_KEYS } from './settings.service';
 
+interface ListViewState {
+
+  title?: string
+
+  documents?: PaperlessDocument[]
+
+  currentPage: number
+  collectionSize: number
+
+  sortField: string
+  sortReverse: boolean
+
+  filterRules: FilterRule[]
+
+  selected?: Set<number>
+
+}
 
 /**
  * This service manages the document list which is displayed using the document list view.
@@ -20,153 +37,174 @@ import { SettingsService, SETTINGS_KEYS } from './settings.service';
 })
 export class DocumentListViewService {
 
-  static DEFAULT_SORT_FIELD = 'created'
-
   isReloading: boolean = false
-  documents: PaperlessDocument[] = []
-  currentPage = 1
+
+  rangeSelectionAnchorIndex: number
+  lastRangeSelectionToIndex: number
+
   currentPageSize: number = this.settings.get(SETTINGS_KEYS.DOCUMENT_LIST_SIZE)
-  collectionSize: number
 
-  /**
-   * This is the current config for the document list. The service will always remember the last settings used for the document list.
-   */
-  private _documentListViewConfig: PaperlessSavedView
-  /**
-   * Optionally, this is the currently selected saved view, which might be null.
-   */
-  private _savedViewConfig: PaperlessSavedView
+  private listViewStates: Map<number, ListViewState> = new Map()
 
-  get savedView(): PaperlessSavedView {
-    return this._savedViewConfig
+  private _activeSavedViewId: number = null
+
+  get activeSavedViewId() {
+    return this._activeSavedViewId
   }
 
-  set savedView(value: PaperlessSavedView) {
-    if (value && !this._savedViewConfig || value && value.id != this._savedViewConfig.id) {
-      //saved view inactive and should be active now, or saved view active, but a different view is requested
-      //this is here so that we don't modify value, which might be the actual instance of the saved view.
-      this.selectNone()
-      this._savedViewConfig = Object.assign({}, value)
-    } else if (this._savedViewConfig && !value) {
-      //saved view active, but document list requested
-      this.selectNone()
-      this._savedViewConfig = null
+  get activeSavedViewTitle() {
+    return this.activeListViewState.title
+  }
+
+  private defaultListViewState(): ListViewState {
+    return {
+      title: null,
+      documents: [],
+      currentPage: 1,
+      collectionSize: null,
+      sortField: "created",
+      sortReverse: true,
+      filterRules: [],
+      selected: new Set<number>()
     }
   }
 
-  get savedViewId() {
-    return this.savedView?.id
+  private get activeListViewState() {
+    if (!this.listViewStates.has(this._activeSavedViewId)) {
+      this.listViewStates.set(this._activeSavedViewId, this.defaultListViewState())
+    }
+    return this.listViewStates.get(this._activeSavedViewId)
   }
 
-  get savedViewTitle() {
-    return this.savedView?.name
-  }
-
-  get documentListView() {
-    return this._documentListViewConfig
-  }
-
-  set documentListView(value) {
-    if (value) {
-      this._documentListViewConfig = Object.assign({}, value)
-      this.saveDocumentListView()
+  activateSavedView(view: PaperlessSavedView) {
+    this.rangeSelectionAnchorIndex = this.lastRangeSelectionToIndex = null
+    if (view) {
+      this._activeSavedViewId = view.id
+      this.loadSavedView(view)
+    } else {
+      this._activeSavedViewId = null
     }
   }
 
-  /**
-   * This is what switches between the saved views and the document list view. Everything on the document list uses
-   * this property to determine the settings for the currently displayed document list.
-   */
-  get view() {
-    return this.savedView || this.documentListView
-  }
-
-  load(view: PaperlessSavedView) {
-    this.documentListView.filter_rules = cloneFilterRules(view.filter_rules)
-    this.documentListView.sort_reverse = view.sort_reverse
-    this.documentListView.sort_field = view.sort_field
-    this.saveDocumentListView()
-  }
-
-  clear() {
-    this.collectionSize = null
-    this.documents = []
-    this.currentPage = 1
+  loadSavedView(view: PaperlessSavedView, closeCurrentView: boolean = false) {
+    if (closeCurrentView) {
+      this._activeSavedViewId = null
+    }
+    this.activeListViewState.filterRules = cloneFilterRules(view.filter_rules)
+    this.activeListViewState.sortField = view.sort_field
+    this.activeListViewState.sortReverse = view.sort_reverse
+    if (this._activeSavedViewId) {
+      this.activeListViewState.title = view.name
+    }
+    this.reduceSelectionToFilter()
   }
 
   reload(onFinish?) {
     this.isReloading = true
+    let activeListViewState = this.activeListViewState
+
     this.documentService.listFiltered(
-      this.currentPage,
+      activeListViewState.currentPage,
       this.currentPageSize,
-      this.view.sort_field,
-      this.view.sort_reverse,
-      this.view.filter_rules).subscribe(
+      activeListViewState.sortField,
+      activeListViewState.sortReverse,
+      activeListViewState.filterRules).subscribe(
         result => {
-          this.collectionSize = result.count
-          this.documents = result.results
+          this.isReloading = false
+          activeListViewState.collectionSize = result.count
+          activeListViewState.documents = result.results
           if (onFinish) {
             onFinish()
           }
-          this.isReloading = false
+          this.rangeSelectionAnchorIndex = this.lastRangeSelectionToIndex = null
         },
         error => {
-          if (this.currentPage != 1 && error.status == 404) {
+          this.isReloading = false
+          if (activeListViewState.currentPage != 1 && error.status == 404) {
             // this happens when applying a filter: the current page might not be available anymore due to the reduced result set.
-            this.currentPage = 1
+            activeListViewState.currentPage = 1
             this.reload()
           }
-          this.isReloading = false
         })
   }
 
   set filterRules(filterRules: FilterRule[]) {
-    //we're going to clone the filterRules object, since we don't
-    //want changes in the filter editor to propagate into here right away.
-    this.view.filter_rules = filterRules
+    this.activeListViewState.filterRules = filterRules
     this.reload()
     this.reduceSelectionToFilter()
     this.saveDocumentListView()
   }
 
   get filterRules(): FilterRule[] {
-    return this.view.filter_rules
+    return this.activeListViewState.filterRules
   }
 
   set sortField(field: string) {
-    this.view.sort_field = field
-    this.saveDocumentListView()
+    this.activeListViewState.sortField = field
     this.reload()
+    this.saveDocumentListView()
   }
 
   get sortField(): string {
-    return this.view.sort_field
+    return this.activeListViewState.sortField
   }
 
   set sortReverse(reverse: boolean) {
-    this.view.sort_reverse = reverse
-    this.saveDocumentListView()
+    this.activeListViewState.sortReverse = reverse
     this.reload()
+    this.saveDocumentListView()
   }
 
   get sortReverse(): boolean {
-    return this.view.sort_reverse
+    return this.activeListViewState.sortReverse
+  }
+
+  get collectionSize(): number {
+    return this.activeListViewState.collectionSize
+  }
+
+  get currentPage(): number {
+    return this.activeListViewState.currentPage
+  }
+
+  set currentPage(page: number) {
+    this.activeListViewState.currentPage = page
+    this.reload()
+    this.saveDocumentListView()
+  }
+
+  get documents(): PaperlessDocument[] {
+    return this.activeListViewState.documents
+  }
+
+  get selected(): Set<number> {
+    return this.activeListViewState.selected
   }
 
   setSort(field: string, reverse: boolean) {
-    this.view.sort_field = field
-    this.view.sort_reverse = reverse
-    this.saveDocumentListView()
+    this.activeListViewState.sortField = field
+    this.activeListViewState.sortReverse = reverse
     this.reload()
+    this.saveDocumentListView()
   }
 
   private saveDocumentListView() {
-    sessionStorage.setItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG, JSON.stringify(this.documentListView))
+    if (this._activeSavedViewId == null) {
+      let savedState: ListViewState = {
+        collectionSize: this.activeListViewState.collectionSize,
+        currentPage: this.activeListViewState.currentPage,
+        filterRules: this.activeListViewState.filterRules,
+        sortField: this.activeListViewState.sortField,
+        sortReverse: this.activeListViewState.sortReverse
+      }
+      sessionStorage.setItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG, JSON.stringify(savedState))
+    }
   }
 
   quickFilter(filterRules: FilterRule[]) {
-    this.savedView = null
-    this.view.filter_rules = filterRules
+    this._activeSavedViewId = null
+    this.activeListViewState.filterRules = filterRules
+    this.activeListViewState.currentPage = 1
     this.reduceSelectionToFilter()
     this.saveDocumentListView()
     this.router.navigate(["documents"])
@@ -214,22 +252,19 @@ export class DocumentListViewService {
     }
   }
 
-  selected = new Set<number>()
-
   selectNone() {
     this.selected.clear()
+    this.rangeSelectionAnchorIndex = this.lastRangeSelectionToIndex = null
   }
 
   reduceSelectionToFilter() {
     if (this.selected.size > 0) {
       this.documentService.listAllFilteredIds(this.filterRules).subscribe(ids => {
-        let subset = new Set<number>()
-        for (let id of ids) {
-          if (this.selected.has(id)) {
-            subset.add(id)
+        for (let id of this.selected) {
+          if (!ids.includes(id)) {
+            this.selected.delete(id)
           }
         }
-        this.selected = subset
       })
     }
   }
@@ -249,29 +284,55 @@ export class DocumentListViewService {
     return this.selected.has(d.id)
   }
 
-  setSelected(d: PaperlessDocument, value: boolean) {
-    if (value) {
-      this.selected.add(d.id)
-    } else if (!value) {
-      this.selected.delete(d.id)
+  toggleSelected(d: PaperlessDocument): void {
+    if (this.selected.has(d.id)) this.selected.delete(d.id)
+    else this.selected.add(d.id)
+    this.rangeSelectionAnchorIndex = this.documentIndexInCurrentView(d.id)
+    this.lastRangeSelectionToIndex = null
+  }
+
+  selectRangeTo(d: PaperlessDocument) {
+    if (this.rangeSelectionAnchorIndex !== null) {
+      const documentToIndex = this.documentIndexInCurrentView(d.id)
+      const fromIndex = Math.min(this.rangeSelectionAnchorIndex, documentToIndex)
+      const toIndex = Math.max(this.rangeSelectionAnchorIndex, documentToIndex)
+
+      if (this.lastRangeSelectionToIndex !== null) {
+        // revert the old selection
+        this.documents.slice(Math.min(this.rangeSelectionAnchorIndex, this.lastRangeSelectionToIndex), Math.max(this.rangeSelectionAnchorIndex, this.lastRangeSelectionToIndex) + 1).forEach(d => {
+          this.selected.delete(d.id)
+        })
+      }
+
+      this.documents.slice(fromIndex, toIndex + 1).forEach(d => {
+        this.selected.add(d.id)
+      })
+      this.lastRangeSelectionToIndex = documentToIndex
+    } else { // e.g. shift key but was first click
+      this.toggleSelected(d)
     }
   }
 
+  documentIndexInCurrentView(documentID: number): number {
+    return this.documents.map(d => d.id).indexOf(documentID)
+  }
+
   constructor(private documentService: DocumentService, private settings: SettingsService, private router: Router) {
-    let documentListViewConfigJson = sessionStorage.getItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG)
+     let documentListViewConfigJson = sessionStorage.getItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG)
     if (documentListViewConfigJson) {
       try {
-        this.documentListView = JSON.parse(documentListViewConfigJson)
+        let savedState: ListViewState = JSON.parse(documentListViewConfigJson)
+        // Remove null elements from the restored state
+        Object.keys(savedState).forEach(k => {
+          if (savedState[k] == null) {
+            delete savedState[k]
+          }
+        })
+        //only use restored state attributes instead of defaults if they are not null
+        let newState = Object.assign(this.defaultListViewState(), savedState)
+        this.listViewStates.set(null, newState)
       } catch (e) {
         sessionStorage.removeItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG)
-        this.documentListView = null
-      }
-    }
-    if (!this.documentListView || this.documentListView.filter_rules == null || this.documentListView.sort_reverse == null || this.documentListView.sort_field == null) {
-      this.documentListView = {
-        filter_rules: [],
-        sort_reverse: true,
-        sort_field: 'created'
       }
     }
   }

src-ui/src/app/services/rest/abstract-name-filter-service.ts

@@ -0,0 +1,14 @@
+import { ObjectWithId } from 'src/app/data/object-with-id'
+import { AbstractPaperlessService } from './abstract-paperless-service'
+
+export abstract class AbstractNameFilterService<T extends ObjectWithId> extends AbstractPaperlessService<T> {
+
+  listFiltered(page?: number, pageSize?: number, sortField?: string, sortReverse?: boolean, nameFilter?: string) {
+    let params = {}
+    if (nameFilter) {
+      params = {'name__icontains': nameFilter}
+    }
+    return this.list(page, pageSize, sortField, sortReverse, params)
+  }
+
+}

src-ui/src/app/services/rest/correspondent.service.ts

@@ -1,12 +1,12 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent';
-import { AbstractPaperlessService } from './abstract-paperless-service';
+import { AbstractNameFilterService } from './abstract-name-filter-service';
 
 @Injectable({
   providedIn: 'root'
 })
-export class CorrespondentService extends AbstractPaperlessService<PaperlessCorrespondent> {
+export class CorrespondentService extends AbstractNameFilterService<PaperlessCorrespondent> {
 
   constructor(http: HttpClient) {
     super(http, 'correspondents')

src-ui/src/app/services/rest/document-type.service.ts

@@ -1,12 +1,12 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 import { PaperlessDocumentType } from 'src/app/data/paperless-document-type';
-import { AbstractPaperlessService } from './abstract-paperless-service';
+import { AbstractNameFilterService } from './abstract-name-filter-service';
 
 @Injectable({
   providedIn: 'root'
 })
-export class DocumentTypeService extends AbstractPaperlessService<PaperlessDocumentType> {
+export class DocumentTypeService extends AbstractNameFilterService<PaperlessDocumentType> {
 
   constructor(http: HttpClient) {
     super(http, 'document_types')

src-ui/src/app/services/rest/document.service.ts

@@ -11,6 +11,7 @@ import { CorrespondentService } from './correspondent.service';
 import { DocumentTypeService } from './document-type.service';
 import { TagService } from './tag.service';
 import { FILTER_RULE_TYPES } from 'src/app/data/filter-rule-type';
+import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions';
 
 export const DOCUMENT_SORT_FIELDS = [
   { field: 'archive_serial_number', name: $localize`ASN` },
@@ -129,4 +130,12 @@ export class DocumentService extends AbstractPaperlessService<PaperlessDocument>
     return this.http.post<SelectionData>(this.getResourceUrl(null, 'selection_data'), {"documents": ids})
   }
 
+  getSuggestions(id: number): Observable<PaperlessDocumentSuggestions> {
+    return this.http.get<PaperlessDocumentSuggestions>(this.getResourceUrl(id, 'suggestions'))
+  }
+
+  bulkDownload(ids: number[], content="both") {
+    return this.http.post(this.getResourceUrl(null, 'bulk_download'), {"documents": ids, "content": content}, { responseType: 'blob' })
+  }
+
 }