Merge remote-tracking branch 'upstream/dev' into fix/mobile-button-group-improvements

.dockerignore

@@ -1,3 +1,4 @@
+**/__pycache__
 /src-ui/.vscode
 /src-ui/node_modules
 /src-ui/dist
@@ -10,3 +11,9 @@
 .pytest_cache
 /dist
 /scripts
+/resources
+**/tests
+**/*.spec.ts
+**/htmlcov
+/src/.pytest_cache
+.idea

.github/workflows/ci.yml

@@ -0,0 +1,252 @@
+name: ci
+
+on: [push, pull_request]
+
+jobs:
+  documentation:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: Persistent Github pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip3.8}
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev
+          pip install --upgrade pipenv
+          pipenv install --system --dev --ignore-pipfile
+      - name: Make documentation
+        run: |
+          cd docs/
+          make html
+      - uses: actions/upload-artifact@v2
+        with:
+          name: documentation
+          path: docs/_build/html/
+
+  tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ['3.6', '3.7', '3.8']
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "${{ matrix.python-version }}"
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: Persistent Github pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip${{ matrix.python }}
+      - name: Tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng
+          pip install --upgrade pipenv
+          pipenv install --system --dev --ignore-pipfile
+          cd src/
+          pytest
+          pycodestyle
+          coveralls
+
+  frontend:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: '15'
+    - name: Build frontend
+      run: ./compile-frontend.sh
+    - name: Upload artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: frontend-compiled
+        path: src/documents/static/frontend/
+
+  build-release:
+    needs: [frontend, documentation, tests]
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        uses: actions/checkout@v2
+      -
+        name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      -
+        name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext liblept5
+          pip3 install -r requirements.txt
+      -
+        name: Download frontend artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: frontend-compiled
+          path: src/documents/static/frontend/
+      -
+        name: Download documentation artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: documentation
+          path: docs/_build/html/
+      -
+        name: Move files
+        run: |
+          mkdir dist
+          mkdir dist/paperless-ng
+          mkdir dist/paperless-ng/docker
+          mkdir dist/paperless-ng/scripts
+          cp .dockerignore .env docker-compose.* Dockerfile paperless.conf.example Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ng/
+          cp docker/*.* dist/paperless-ng/docker/
+          cp scripts/*.service scripts/*.sh dist/paperless-ng/scripts/
+          cp src/ dist/paperless-ng/src -r
+          cp docs/_build/html/ dist/paperless-ng/docs -r
+      -
+        name: Compile messages
+        run: |
+          cd dist/paperless-ng/src
+          python3 manage.py compilemessages
+      -
+        name: Collect static files
+        run: |
+          cd dist/paperless-ng/src
+          python3 manage.py collectstatic --no-input
+      -
+        name: Make release package
+        run: |
+          cd dist
+          find . -name __pycache__ | xargs rm -r
+          tar -cJf paperless-ng.tar.xz paperless-ng/
+      -
+        name: Upload release artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: release
+          path: dist/paperless-ng.tar.xz
+
+  publish-release:
+    runs-on: ubuntu-latest
+    needs: build-release
+    if: contains(github.ref, 'refs/tags/ng-')
+    steps:
+      -
+        name: Download release artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: release
+          path: ./
+      -
+        name: Get branch name
+        id: get_branch_name
+        run: |
+          echo ::set-output name=branch_name::${GITHUB_REF#refs/tags/ng-}
+      -
+        name: Create release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.get_branch_name.outputs.branch_name }}
+          release_name: Paperless-ng ${{ steps.get_branch_name.outputs.branch_name }}
+          draft: false
+          prerelease: true
+          body: |
+            TODO: Insert changelog here (it's also in the documentation)
+      -
+        name: Upload release archive
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: ./paperless-ng.tar.xz
+          asset_name: paperless-ng-${{ steps.get_branch_name.outputs.branch_name }}.tar.xz
+          asset_content_type: application/x-xz
+
+  # build and push image to docker hub.
+  build-docker-image:
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*')
+    runs-on: ubuntu-latest
+    needs: [frontend, tests]
+    steps:
+      -
+        name: Prepare
+        id: prepare
+        run: |
+          VERSION=edge
+          if [[ $GITHUB_REF == refs/tags/ng-* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/ng-}
+          fi
+          if [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=${GITHUB_REF#refs/heads/}
+          fi
+          echo ::set-output name=version::${VERSION}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Download frontend artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: frontend-compiled
+          path: src/documents/static/frontend/
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          push: true
+          tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+      -
+        name: Inspect image
+        run: |
+          docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }}

.gitignore

@@ -65,8 +65,8 @@ target/
 .virtualenv
 virtualenv
 /venv
-docker-compose.env
+/docker-compose.env
-docker-compose.yml
+/docker-compose.yml
 
 # Used for development
 scripts/import-for-development

.travis.yml

@@ -1,51 +0,0 @@
-language: python
-
-dist: focal
-os: linux
-
-jobs:
-  include:
-    - name: "Paperless on Python 3.6"
-      python: "3.6"
-
-    - name: "Paperless on Python 3.7"
-      python: "3.7"
-
-    - name: "Paperless on Python 3.8"
-      python: "3.8"
-
-    - name: "Documentation"
-      script:
-        - cd docs/
-        - make html
-      after_success: true
-
-    - name: "Front end"
-      language: node_js
-      node_js:
-        - 15
-      before_install: true
-      install:
-        - cd src-ui/
-        - npm install -g @angular/cli
-        - npm install
-      script:
-        - ng build --prod
-      after_success: true
-
-
-before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng
-
-install:
-  - pip install --upgrade pipenv
-  - pipenv install --system --dev
-
-script:
-  - cd src/
-  - pipenv run pytest --cov
-  - pipenv run pycodestyle
-
-after_success:
-  - pipenv run coveralls

Dockerfile

@@ -1,45 +1,62 @@
+FROM ubuntu:20.04 AS jbig2enc
+
+WORKDIR /usr/src/jbig2enc
+
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential automake libtool libleptonica-dev zlib1g-dev git ca-certificates
+
+RUN git clone https://github.com/agl/jbig2enc .
+RUN ./autogen.sh
+RUN ./configure && make
+
 FROM python:3.7-slim
 
 WORKDIR /usr/src/paperless/
 
 COPY requirements.txt ./
 
-#Dependencies
+# Binary dependencies
 RUN apt-get update \
   && apt-get -y --no-install-recommends install \
-		build-essential \
+  	# Basic dependencies
 		curl \
 		file \
 		fonts-liberation \
 		gettext \
-		ghostscript \
 		gnupg \
-		icc-profiles-free \
 		imagemagick \
-		libatlas-base-dev \
-		liblept5 \
-		libmagic-dev \
-		libpoppler-cpp-dev \
-		libpq-dev \
-		libqpdf-dev \
-		libxml2 \
 		libxslt1-dev \
 		mime-support \
 		optipng \
+		sudo \
+		tzdata \
+  	# OCRmyPDF dependencies
+		ghostscript \
+		icc-profiles-free \
+		liblept5 \
+		libxml2 \
 		pngquant \
 		qpdf \
-		sudo \
 		tesseract-ocr \
 		tesseract-ocr-eng \
 		tesseract-ocr-deu \
 		tesseract-ocr-fra \
 		tesseract-ocr-ita \
 		tesseract-ocr-spa \
-		tzdata \
 		unpaper \
 		zlib1g \
-	&& pip3 install --upgrade supervisor setuptools \
+		&& rm -rf /var/lib/apt/lists/*
-	&& pip install --no-cache-dir -r requirements.txt \
+
+# Python dependencies
+RUN apt-get update \
+  && apt-get -y --no-install-recommends install \
+		build-essential \
+		libatlas-base-dev \
+		libmagic-dev \
+		libpoppler-cpp-dev \
+		libpq-dev \
+		libqpdf-dev \
+	&& python3 -m pip install --upgrade --no-cache-dir supervisor \
+  && python3 -m pip install --no-cache-dir -r requirements.txt \
 	&& apt-get -y purge build-essential libqpdf-dev \
 	&& apt-get -y autoremove --purge \
 	&& rm -rf /var/lib/apt/lists/* \
@@ -52,6 +69,12 @@ COPY docker/gunicorn.conf.py ./
 COPY docker/supervisord.conf /etc/supervisord.conf
 COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh
 
+# copy jbig2enc
+COPY --from=jbig2enc /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
+COPY --from=jbig2enc /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
+COPY --from=jbig2enc /usr/src/jbig2enc/src/*.h /usr/local/include/
+
+
 # copy app
 COPY src/ ./src/
 

Pipfile

@@ -8,9 +8,6 @@ url = "https://www.piwheels.org/simple"
 verify_ssl = true
 name = "piwheels"
 
-[requires]
-python_version = "3.6"
-
 [packages]
 dateparser = "~=0.7.6"
 django = "~=3.1.3"
@@ -26,7 +23,8 @@ imap-tools = "*"
 langdetect = "*"
 pdftotext = "*"
 pathvalidate = "*"
-pillow = "*"
+# pinned to 8.1.0, since aarch64 wheels might not be available beyond that
+pillow = "==8.1.0"
 pikepdf = "*"
 python-gnupg = "*"
 python-dotenv = "*"

Pipfile.lock

@@ -1,12 +1,10 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c35d84fd7f4f1c7d599039712362935e7c41a226b0ab3d83d8c1c2fb2ad0962a"
+            "sha256": "27c8973d55150371bd18406cc90e61ad4a7ec4b9353007045d611a87120a2703"
         },
         "pipfile-spec": 6,
-        "requires": {
+        "requires": {},
-            "python_version": "3.6"
-        },
         "sources": [
             {
                 "name": "pypi",
@@ -439,6 +437,7 @@
         "pillow": {
             "hashes": [
                 "sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6",
+                "sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded",
                 "sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865",
                 "sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174",
                 "sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032",
@@ -464,9 +463,12 @@
                 "sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8",
                 "sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59",
                 "sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d",
+                "sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7",
                 "sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a",
+                "sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0",
                 "sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b",
-                "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d"
+                "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d",
+                "sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae"
             ],
             "index": "pypi",
             "version": "==8.1.0"
@@ -1375,6 +1377,15 @@
             "index": "pypi",
             "version": "==3.20.1"
         },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.7.4.3"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",

compile-frontend.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+cd src-ui
+npm install
+./node_modules/.bin/ng build --prod

docker-compose.postgres.yml

@@ -2,11 +2,11 @@ version: "3.4"
 services:
   broker:
     image: redis:6.0
-    restart: always
+    restart: unless-stopped
 
   db:
     image: postgres:13
-    restart: always
+    restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
     environment:
@@ -16,7 +16,7 @@ services:
 
   webserver:
     build: .
-    restart: always
+    restart: unless-stopped
     depends_on:
       - db
       - broker

docker-compose.sqlite.yml

@@ -2,11 +2,11 @@ version: "3.4"
 services:
   broker:
     image: redis:6.0
-    restart: always
+    restart: unless-stopped
 
   webserver:
     build: .
-    restart: always
+    restart: unless-stopped
     depends_on:
       - broker
     ports:

docker-compose.tika.yml

@@ -2,11 +2,11 @@ version: "3.4"
 services:
   broker:
     image: redis:6.0
-    restart: always
+    restart: unless-stopped
 
   webserver:
     build: .
-    restart: always
+    restart: unless-stopped
     depends_on:
       - broker
     ports:

docker/hub/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=paperless

docker/hub/docker-compose.env

@@ -0,0 +1,34 @@
+# The UID and GID of the user used to run paperless in the container. Set this
+# to your UID and GID on the host so that you have write access to the
+# consumption directory.
+#USERMAP_UID=1000
+#USERMAP_GID=1000
+
+# Additional languages to install for text recognition, separated by a
+# whitespace. Note that this is
+# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the
+# default language used when guessing the language from the OCR output.
+# The container installs English, German, Italian, Spanish and French by
+# default.
+# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster
+# for available languages.
+#PAPERLESS_OCR_LANGUAGES=tur ces
+
+###############################################################################
+# Paperless-specific settings                                                 #
+###############################################################################
+
+# All settings defined in the paperless.conf.example can be used here. The
+# Docker setup does not use the configuration file.
+# A few commonly adjusted settings are provided below.
+
+# Adjust this key if you plan to make paperless available publicly. It should
+# be a very long sequence of random characters. You don't need to remember it.
+#PAPERLESS_SECRET_KEY=change-me
+
+# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC.
+#PAPERLESS_TIME_ZONE=America/Los_Angeles
+
+# The default language to use for OCR. Set this to the language most of your
+# documents are written in.
+#PAPERLESS_OCR_LANGUAGE=eng

docker/hub/docker-compose.postgres.yml

@@ -2,11 +2,11 @@ version: "3.4"
 services:
   broker:
     image: redis:6.0
-    restart: always
+    restart: unless-stopped
 
   db:
     image: postgres:13
-    restart: always
+    restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
     environment:
@@ -15,8 +15,8 @@ services:
       POSTGRES_PASSWORD: paperless
 
   webserver:
-    image: jonaswinkler/paperless-ng:0.9.14
+    image: jonaswinkler/paperless-ng:latest
-    restart: always
+    restart: unless-stopped
     depends_on:
       - db
       - broker

docker/hub/docker-compose.sqlite.yml

@@ -2,11 +2,11 @@ version: "3.4"
 services:
   broker:
     image: redis:6.0
-    restart: always
+    restart: unless-stopped
 
   webserver:
-    image: jonaswinkler/paperless-ng:0.9.14
+    image: jonaswinkler/paperless-ng:latest
-    restart: always
+    restart: unless-stopped
     depends_on:
       - broker
     ports:

docker/hub/docker-compose.tika.yml

@@ -2,11 +2,11 @@ version: "3.4"
 services:
   broker:
     image: redis:6.0
-    restart: always
+    restart: unless-stopped
 
   webserver:
-    image: jonaswinkler/paperless-ng:0.9.14
+    image: jonaswinkler/paperless-ng:latest
-    restart: always
+    restart: unless-stopped
     depends_on:
       - broker
     ports:

docs/administration.rst

@@ -48,12 +48,9 @@ Options available to bare-metal and non-docker installations:
 Restoring
 =========
 
-
-
-
 .. _administration-updating:
 
-Updating paperless
+Updating Paperless
 ##################
 
 If a new release of paperless-ng is available, upgrading depends on how you
@@ -69,51 +66,29 @@ First of all, ensure that paperless is stopped.
 
 After that, :ref:`make a backup <administration-backup>`.
 
-A.  If you used the dockerfiles archive, simply download the files of the new release,
+A.  If you pull the image from the docker hub, all you need to do is:
-    adjust the settings in the files (i.e., the path to your consumption directory),
-    and replace your existing docker-compose files. Then start paperless as usual,
-    which will pull the new image, and update your database, if necessary:
 
     .. code:: shell-session
 
-        $ cd /path/to/paperless
+        $ docker-compose pull
         $ docker-compose up
     
-    If you see everything working, you can start paperless-ng with "-d" to have it
+    The docker-compose files refer to the ``latest`` version, which is always the latest
-    run in the background.
+    stable release.
         
-    .. hint::
+B.  If you built the image yourself, do the following:
-
-        The released docker-compose files specify exact versions to be pulled from the hub.
-        This is to ensure that if the docker-compose files should change at some point
-        (i.e., services updates/configured differently), you wont run into trouble due to
-        docker pulling the ``latest`` image and running it in an older environment.
-        
-B.  If you built the image yourself, grab the new archive and replace your current
-    paperless folder with the new contents.
-
-    After that, make the necessary adjustments to the docker-compose.yml (i.e.,
-    adjust your consumption directory).
-
-    Build and start the new image with:
 
     .. code:: shell-session
 
-        $ cd /path/to/paperless
+        $ git pull
+        $ ./compile-frontend.sh
         $ docker-compose build
         $ docker-compose up
 
+Running `docker-compose up` will also apply any new database migrations.
 If you see everything working, you can start paperless-ng with "-d" to have it
 run in the background.
 
-.. hint::
-
-    You can usually keep your ``docker-compose.env`` file, since this file will
-    never include mandatory configuration options. However, it is worth checking
-    out the new version of this file, since it might have new recommendations
-    on what to configure.
-
-
 Updating paperless without docker
 =================================
 
@@ -135,26 +110,16 @@ After grabbing the new release and unpacking the contents, do the following:
     This creates a new virtual environment (or uses your existing environment)
     and installs all dependencies into it.
     
-3.  Collect static files.
+3.  Migrate the database.
-
-    .. code:: shell-session
-
-        $ cd src
-        $ pipenv run python3 manage.py collectstatic --clear
-    
-4.  Migrate the database.
 
     .. code:: shell-session
 
         $ cd src
         $ pipenv run python3 manage.py migrate
     
-5.  Update translation files.
+    This might not actually do anything. Not every new paperless version comes with new
+    database migrations.
     
-    .. code:: shell-session
-
-        $ cd src
-        $ pipenv run python3 manage.py compilemessages
         
 Management utilities
 ####################

docs/setup.rst

@@ -3,35 +3,6 @@
 Setup
 *****
 
-Download
-########
-
-Go to the project page on GitHub and download the
-`latest release <https://github.com/jonaswinkler/paperless-ng/releases>`_.
-There are multiple options available.
-
-*   Download the dockerfiles archive if you want to pull paperless from
-    Docker Hub.
-
-*   Download the dist archive and extract it if you want to build the docker image
-    yourself or want to install paperless without docker.
-
-.. hint::
-
-    In contrast to paperless, the recommended way to get and update paperless-ng
-    is not to pull the entire git repository. Paperless-ng includes artifacts
-    that need to be compiled, and that's already done for you in the release.
-
-.. admonition:: Want to try out paperless-ng before migrating?
-
-    The release contains a file ``.env`` which sets the docker-compose project
-    name to "paperless", which is the same as before and instructs docker-compose
-    to reuse and upgrade your paperless volumes.
-
-    Just rename the project name in that file to anything else and docker-compose
-    will create fresh volumes for you!
-
-
 Overview of Paperless-ng
 ########################
 
@@ -110,22 +81,34 @@ Installation
 
 You can go multiple routes with setting up and running Paperless:
 
-* The `docker route`_
+* :ref:`Pull the image from Docker Hub <setup-docker_hub>`
-* The `bare metal route`_
+* :ref:`Build the Docker image yourself <setup-docker_build>`
+* :ref:`Install Paperless directly on your system (bare metal) <setup-bare_metal>`
 
-The `docker route`_ is quick & easy. This is the recommended route. This configures all the stuff
+The Docker routes are quick & easy. These are the recommended routes. This configures all the stuff
 from above automatically so that it just works and uses sensible defaults for all configuration options.
 
-The `bare metal route`_ is more complicated to setup but makes it easier
+The bare metal route is more complicated to setup but makes it easier
 should you want to contribute some code back. You need to configure and
 run the above mentioned components yourself.
 
-.. _setup-docker_route:
+.. _setup-docker_hub:
 
-Docker Route
+Install Paperless from Docker Hub
-============
+=================================
 
-1.  Install `Docker`_ and `docker-compose`_. [#compose]_
+1.  Go to the `/docker/hub directory on the project page <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/hub>`_
+    and download one of the ``docker-compose.*.yml`` files, depending on which database backend you
+    want to use. Rename this file to `docker-compose.yml`.
+    Download the ``docker-compose.env`` file and the ``.env`` file as well and store them
+    in the same directory.
+
+    .. hint::
+
+        For new installations, it is recommended to use PostgreSQL as the database
+        backend.
+
+2.  Install `Docker`_ and `docker-compose`_.
 
     .. caution::
 
@@ -142,15 +125,7 @@ Docker Route
         .. _Docker installation guide: https://docs.docker.com/engine/installation/
         .. _docker-compose installation guide: https://docs.docker.com/compose/install/
 
-2.  Copy either ``docker-compose.sqlite.yml`` or ``docker-compose.postgres.yml`` to
+3.  Modify ``docker-compose.yml`` to your preferences. You may want to change the path
-    ``docker-compose.yml``, depending on which database backend you want to use.
-
-    .. hint::
-
-        For new installations, it is recommended to use PostgreSQL as the database
-        backend.
-
-2.  Modify ``docker-compose.yml`` to your preferences. You may want to change the path
     to the consumption directory in this file. Find the line that specifies where
     to mount the consumption directory:
 
@@ -167,7 +142,7 @@ Docker Route
     Don't change the part after the colon or paperless wont find your documents.
 
 
-3.  Modify ``docker-compose.env``, following the comments in the file. The
+4.  Modify ``docker-compose.env``, following the comments in the file. The
     most important change is to set ``USERMAP_UID`` and ``USERMAP_GID``
     to the uid and gid of your user on the host system. This ensures that
     both the docker container and you on the host machine have write access
@@ -177,7 +152,7 @@ Docker Route
 
     .. note::
 
-        You can use any settings from the file ``paperless.conf`` in this file.
+        You can use any settings from the file ``paperless.conf.example`` in this file.
         Have a look at :ref:`configuration` to see whats available.
 
     .. caution::
@@ -188,11 +163,10 @@ Docker Route
         with the default configuration. You will need to use ``PAPERLESS_CONSUMER_POLLING``,
         which will disable inotify. See :ref:`here <configuration-polling>`.
 
-4.  Run ``docker-compose up -d``. This will create and start the necessary
+5.  Run ``docker-compose up -d``. This will create and start the necessary
-    containers. This will also build the image of paperless if you grabbed the
+    containers.
-    source archive.
 
-5.  To be able to login, you will need a super user. To create it, execute the
+6.  To be able to login, you will need a super user. To create it, execute the
     following command:
 
     .. code-block:: shell-session
@@ -202,7 +176,7 @@ Docker Route
     This will prompt you to set a username, an optional e-mail address and
     finally a password.
 
-6.  The default ``docker-compose.yml`` exports the webserver on your local port
+7.  The default ``docker-compose.yml`` exports the webserver on your local port
     8000. If you haven't adapted this, you should now be able to visit your
     Paperless instance at ``http://127.0.0.1:8000``. You can login with the
     user and password you just created.
@@ -210,11 +184,35 @@ Docker Route
 .. _Docker: https://www.docker.com/
 .. _docker-compose: https://docs.docker.com/compose/install/
 
-.. [#compose] You of course don't have to use docker-compose, but it
+.. _setup-docker_build:
-   simplifies deployment immensely. If you know your way around Docker, feel
-   free to tinker around without using compose!
 
-.. _`setup-bare_metal`:
+Build the docker image yourself
+===============================
+
+1.  Clone the entire repository of paperless:
+
+    .. code:: shell-session
+
+        git clone https://github.com/jonaswinkler/paperless-ng
+
+    The master branch always reflects the latest stable version.
+
+2.  Copy one of the ``docker-compose.*.yml`` to ``docker-compose.yml``,
+    depending on which database backend you want to use. Copy
+    ``docker-compose.env.example`` to ``docker-compose.env``.
+
+3.  Run the `compile-frontend.sh` script. This requires `node` and `npm`.
+
+4.  Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run
+    ``docker-compose up -d`` to start the containers, do
+
+    .. code:: shell-session
+
+        $ docker-compose build
+
+    before that to build the image.
+
+.. _setup-bare_metal:
 
 Bare Metal Route
 ================
@@ -229,7 +227,6 @@ writing. Windows is not and will never be supported.
     *   ``python3-pip``, optionally ``pipenv`` for package installation
     *   ``python3-dev``
 
-    *   ``gettext`` for compiling interface translations
     *   ``fonts-liberation`` for generating thumbnails for plain text files
     *   ``imagemagick`` >= 6 for PDF conversion
     *   ``optipng`` for optimizing thumbnails
@@ -259,8 +256,9 @@ writing. Windows is not and will never be supported.
 3.  Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
     to use PostgreSQL, SQLite is avialable as well.
 
-4.  Get the release archive. If you pull the git repo as it is, you also have to compile the front end by yourself.
+4.  Get the release archive from `<https://github.com/jonaswinkler/paperless-ng/releases>`_.
-    Extract the frontend to a place from where you wish to execute it, such as ``/opt/paperless``.
+    If you clone the git repo as it is, you also have to compile the front end by yourself.
+    Extract the archive to a place from where you wish to execute it, such as ``/opt/paperless``.
 
 5.  Configure paperless. See :ref:`configuration` for details. Edit the included ``paperless.conf`` and adjust the
     settings to your needs. Required settings for getting paperless running are:
@@ -296,15 +294,9 @@ writing. Windows is not and will never be supported.
 
     .. code:: bash
 
-        # This collects static files from paperless and django.
-        python3 manage.py collectstatic --clear --no-input
-        
         # This creates the database schema.
         python3 manage.py migrate
 
-        # This creates the translation files for paperless.
-        python3 manage.py compilemessages
-
         # This creates your first paperless user
         python3 manage.py createsuperuser
 
@@ -374,6 +366,11 @@ writing. Windows is not and will never be supported.
 
         <policy domain="coder" rights="read|write" pattern="PDF" />
 
+13. Optional: Install the `jbig2enc <https://ocrmypdf.readthedocs.io/en/latest/jbig2.html>`_
+    encoder. This will reduce the size of generated PDF documents. You'll most likely need
+    to compile this by yourself, because this software has been patented until around 2017 and
+    binary packages are not available for most distributions.
+
 Migration to paperless-ng
 #########################
 
@@ -407,13 +404,14 @@ Migration to paperless-ng is then performed in a few simple steps:
     paperless.
 
 3.  Download the latest release of paperless-ng. You can either go with the
-    docker-compose files or use the archive to build the image yourself.
+    docker-compose files from `here <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/hub>`_
+    or clone the repository to build the image yourself.
     You can either replace your current paperless folder or put paperless-ng
     in a different location.
 
     .. caution::
 
-        The release include a ``.env`` file. This will set the
+        Paperless includes a ``.env`` file. This will set the
         project name for docker compose to ``paperless`` so that paperless-ng will
         automatically reuse your existing paperless volumes. When you start it, it
         will migrate your existing data. After that, your old paperless installation
@@ -423,16 +421,10 @@ Migration to paperless-ng is then performed in a few simple steps:
     If you want to switch to PostgreSQL, do that after you migrated your existing
     SQLite database.
 
-5.  Adjust ``docker-compose.yml`` and
+5.  Adjust ``docker-compose.yml`` and ``docker-compose.env`` to your needs.
-    ``docker-compose.env`` to your needs.
+    See :ref:`setup-docker_hub` for details on which edits are advised.
-    See `docker route`_ for details on which edits are advised.
 
-6.  Since ``docker-compose`` would just use the the old paperless image, we need to
+6.  :ref:`Update paperless. <administration-updating>`
-    manually build a new image:
-
-    .. code:: shell-session
-
-        $ docker-compose build
 
 7.  In order to find your existing documents with the new search feature, you need
     to invoke a one-time operation that will create the search index:
@@ -453,7 +445,7 @@ Migration to paperless-ng is then performed in a few simple steps:
     This will run paperless in the background and automatically start it on system boot.
 
 9.  Paperless installed a permanent redirect to ``admin/`` in your browser. This
-    redirect is still in place and prevents access to the new UI. Clear
+    redirect is still in place and prevents access to the new UI. Clear your
     browsing cache in order to fix this.
 
 10.  Optionally, follow the instructions below to migrate your existing data to PostgreSQL.
@@ -605,6 +597,4 @@ For details, refer to :ref:`configuration`.
     The actual matching of the algorithm is fast and works on Raspberry Pi as
     well as on any other device.
 
-
-
 .. _redis: https://redis.io/

docs/troubleshooting.rst

@@ -75,6 +75,6 @@ You might encounter errors such as:
 
 This happens when paperless does not have permission to delete files inside the consumption directory.
 Ensure that ``USERMAP_UID`` and ``USERMAP_GID`` are set to the user id and group id you use on the host operating system, if these are
-different from ``1000``. See :ref:`setup-docker_route`.
+different from ``1000``. See :ref:`setup-docker_hub`.
 
 Also ensure that you are able to read and write to the consumption directory on the host.

docs/usage_overview.rst

@@ -118,7 +118,7 @@ to share any documents with paperless. This can be combined with any of the mobi
 scanning apps out there, such as Office Lens.
 
 Furthermore, there is the  `Paperless App <https://github.com/bauerj/paperless_app>`_ as well,
-which no only has document upload, but also document editing and browsing.
+which not only has document upload, but also document browsing and download features.
 
 .. _usage-email:
 

requirements.txt

@@ -0,0 +1,74 @@
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
+-i https://pypi.python.org/simple
+--extra-index-url https://www.piwheels.org/simple
+arrow==0.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+asgiref==3.3.1; python_version >= '3.5'
+blessed==1.17.12
+certifi==2020.12.5
+cffi==1.14.4
+chardet==4.0.0; python_version >= '3.1'
+coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+dateparser==0.7.6
+django-cors-headers==3.6.0
+django-extensions==3.1.0
+django-filter==2.4.0
+django-picklefield==3.0.1; python_version >= '3'
+django-q==1.3.4
+django==3.1.5
+djangorestframework==3.12.2
+filelock==3.0.12
+fuzzywuzzy==0.18.0
+gunicorn==20.0.4
+humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+imap-tools==0.34.0
+img2pdf==0.4.0
+importlib-metadata==3.3.0; python_version < '3.8'
+inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+inotifyrecursive==0.3.5
+joblib==1.0.0; python_version >= '3.6'
+langdetect==1.0.8
+lxml==4.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+numpy==1.19.5; python_version >= '3.6'
+ocrmypdf==11.4.5
+pathvalidate==2.3.2
+pdfminer.six==20201018; python_version >= '3.4'
+pdftotext==2.1.5
+pikepdf==2.2.5
+pillow==8.1.0
+pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+psycopg2-binary==2.8.6
+pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.8.1
+python-dotenv==0.15.0
+python-gnupg==0.4.6
+python-levenshtein==0.12.0
+python-magic==0.4.18
+pytz==2020.5
+redis==3.5.3
+regex==2020.11.13
+reportlab==3.5.59
+requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+scikit-learn==0.23.2
+scipy==1.5.4; python_version >= '3.6'
+six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sortedcontainers==2.3.0
+sqlparse==0.4.1; python_version >= '3.5'
+threadpoolctl==2.1.0; python_version >= '3.5'
+tika==1.24
+tqdm==4.55.1
+typing-extensions==3.7.4.3; python_version < '3.8'
+tzlocal==2.1
+urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+watchdog==1.0.2
+wcwidth==0.2.5
+whitenoise==5.2.0
+whoosh==2.7.4
+zipp==3.4.0; python_version >= '3.6'

scripts/make-release.sh

@@ -1,125 +0,0 @@
-#!/bin/bash
-
-# Release checklist
-# - wait for travis build.
-# adjust src/paperless/version.py
-# changelog in the documentation
-# adjust versions in docker/hub/*
-# adjust version in src-ui/src/environments/prod
-# If docker-compose was modified: all compose files are the same.
-
-# Steps:
-# run release script "dev", push
-# if it works: new tag, merge into master
-# on master: make release "lastest", push
-# on master: make release "version-tag", push
-# publish release files
-
-set -e
-
-
-VERSION=$1
-
-if [ -z "$VERSION" ]
-then
-	echo "Need a version string."
-	exit 1
-fi
-
-# source root directory of paperless
-PAPERLESS_ROOT=$(git rev-parse --show-toplevel)
-
-# output directory
-PAPERLESS_DIST="$PAPERLESS_ROOT/dist"
-PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng"
-PAPERLESS_DIST_DOCKERFILES="$PAPERLESS_DIST/paperless-ng-dockerfiles"
-
-if [ -d "$PAPERLESS_DIST" ]
-then
-	echo "Removing $PAPERLESS_DIST"
-	rm "$PAPERLESS_DIST" -r
-fi
-
-mkdir "$PAPERLESS_DIST"
-mkdir "$PAPERLESS_DIST_APP"
-mkdir "$PAPERLESS_DIST_APP/docker"
-mkdir "$PAPERLESS_DIST_APP/scripts"
-mkdir "$PAPERLESS_DIST_DOCKERFILES"
-
-# setup dependencies.
-
-cd "$PAPERLESS_ROOT"
-
-pipenv clean
-pipenv install --dev
-pipenv lock --keep-outdated -r > "$PAPERLESS_DIST_APP/requirements.txt"
-
-# test if the application works.
-
-cd "$PAPERLESS_ROOT/src"
-#pipenv run pytest --cov
-#pipenv run pycodestyle
-
-# make the documentation.
-
-cd "$PAPERLESS_ROOT/docs"
-make clean html
-
-# copy stuff into place
-
-# the application itself
-
-cp "$PAPERLESS_ROOT/.env" \
-  "$PAPERLESS_ROOT/.dockerignore" \
-	"$PAPERLESS_ROOT/CONTRIBUTING.md" \
-	"$PAPERLESS_ROOT/LICENSE" \
-	"$PAPERLESS_ROOT/Pipfile" \
-	"$PAPERLESS_ROOT/Pipfile.lock" \
-	"$PAPERLESS_ROOT/README.md" "$PAPERLESS_DIST_APP"
-
-cp "$PAPERLESS_ROOT/paperless.conf.example" "$PAPERLESS_DIST_APP/paperless.conf"
-
-# copy python source, templates and static files.
-cd "$PAPERLESS_ROOT"
-find src -name '*.po' -o -wholename '*/templates/*' -o -wholename '*/static/*' -o -name '*.py' | cpio -pdm "$PAPERLESS_DIST_APP"
-
-# build the front end.
-
-cd "$PAPERLESS_ROOT/src-ui"
-ng build --prod --output-hashing none --sourceMap=false --output-path "$PAPERLESS_DIST_APP/src/documents/static/frontend"
-
-# documentation
-cp "$PAPERLESS_ROOT/docs/_build/html/" "$PAPERLESS_DIST_APP/docs" -r
-
-# docker files for building the image yourself
-cp "$PAPERLESS_ROOT/docker/local/"* "$PAPERLESS_DIST_APP"
-cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_APP"
-
-# docker files for pulling from docker hub
-cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST_DOCKERFILES"
-cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST_DOCKERFILES"
-cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_DOCKERFILES"
-
-# auxiliary files required for the docker image
-cp "$PAPERLESS_ROOT/docker/docker-entrypoint.sh" "$PAPERLESS_DIST_APP/docker/"
-cp "$PAPERLESS_ROOT/docker/gunicorn.conf.py" "$PAPERLESS_DIST_APP/docker/"
-cp "$PAPERLESS_ROOT/docker/imagemagick-policy.xml" "$PAPERLESS_DIST_APP/docker/"
-cp "$PAPERLESS_ROOT/docker/supervisord.conf" "$PAPERLESS_DIST_APP/docker/"
-
-# auxiliary files for bare metal installs
-cp "$PAPERLESS_ROOT/scripts/paperless-webserver.service" "$PAPERLESS_DIST_APP/scripts/"
-cp "$PAPERLESS_ROOT/scripts/paperless-consumer.service" "$PAPERLESS_DIST_APP/scripts/"
-cp "$PAPERLESS_ROOT/scripts/paperless-scheduler.service" "$PAPERLESS_DIST_APP/scripts/"
-
-# try to make the docker build.
-
-cd "$PAPERLESS_DIST_APP"
-
-docker build . -t "jonaswinkler/paperless-ng:$VERSION"
-
-# works. package the app!
-
-cd "$PAPERLESS_DIST"
-
-tar -cJf "paperless-ng-$VERSION.tar.xz" paperless-ng/
-tar -cJf "paperless-ng-$VERSION-dockerfiles.tar.xz" paperless-ng-dockerfiles/

scripts/push-release.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -e
-
-
-VERSION=$1
-
-if [ -z "$VERSION" ]
-then
-	echo "Need a version string."
-	exit 1
-fi
-
-# source root directory of paperless
-PAPERLESS_ROOT=$(git rev-parse --show-toplevel)
-
-# output directory
-PAPERLESS_DIST="$PAPERLESS_ROOT/dist"
-PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng"
-
-cd "$PAPERLESS_DIST_APP"
-
-docker push "jonaswinkler/paperless-ng:$VERSION"

src-ui/angular.json

@@ -58,6 +58,7 @@
 									"with": "src/environments/environment.prod.ts"
 								}
 							],
+							"outputPath": "../src/documents/static/frontend/",
 							"optimization": true,
 							"outputHashing": "none",
 							"sourceMap": false,

src/paperless_mail/tests/test_mail.py

@@ -41,7 +41,7 @@ class BogusMailBox(ContextManager):
 
     folder = BogusFolderManager()
 
-    def fetch(self, criteria, mark_seen):
+    def fetch(self, criteria, mark_seen, charset=""):
         msg = self.messages
 
         criteria = str(criteria).strip('()').split(" ")