paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-02 13:45:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into fix/mobile-button-group-improvements

Browse Source
This commit is contained in:
Michael Shamoon 2021-01-11 16:16:12 -08:00
commit 95d9c8fa2b
26 changed files with 554 additions and 390 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.dockerignore
View File

@ -1,3 +1,4 @@
**/__pycache__
/src-ui/.vscode
/src-ui/node_modules
/src-ui/dist
@ -10,3 +11,9 @@
.pytest_cache
/dist
/scripts
/resources
**/tests
**/*.spec.ts
**/htmlcov
/src/.pytest_cache
.idea

252
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,252 @@
name: ci
on: [push, pull_request]
jobs:
documentation:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Get pip cache dir
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Persistent Github pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip3.8}
- name: Install dependencies
run: |
sudo apt-get update -qq
sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev
pip install --upgrade pipenv
pipenv install --system --dev --ignore-pipfile
- name: Make documentation
run: |
cd docs/
make html
- uses: actions/upload-artifact@v2
with:
name: documentation
path: docs/_build/html/
tests:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ['3.6', '3.7', '3.8']
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "${{ matrix.python-version }}"
- name: Get pip cache dir
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Persistent Github pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip${{ matrix.python }}
- name: Tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
sudo apt-get update -qq
sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng
pip install --upgrade pipenv
pipenv install --system --dev --ignore-pipfile
cd src/
pytest
pycodestyle
coveralls
frontend:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: '15'
- name: Build frontend
run: ./compile-frontend.sh
- name: Upload artifact
uses: actions/upload-artifact@v2
with:
name: frontend-compiled
path: src/documents/static/frontend/
build-release:
needs: [frontend, documentation, tests]
runs-on: ubuntu-20.04
steps:
-
uses: actions/checkout@v2
-
name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.7
-
name: Install dependencies
run: |
sudo apt-get update -qq
sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext liblept5
pip3 install -r requirements.txt
-
name: Download frontend artifact
uses: actions/download-artifact@v2
with:
name: frontend-compiled
path: src/documents/static/frontend/
-
name: Download documentation artifact
uses: actions/download-artifact@v2
with:
name: documentation
path: docs/_build/html/
-
name: Move files
run: |
mkdir dist
mkdir dist/paperless-ng
mkdir dist/paperless-ng/docker
mkdir dist/paperless-ng/scripts
cp .dockerignore .env docker-compose.* Dockerfile paperless.conf.example Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ng/
cp docker/*.* dist/paperless-ng/docker/
cp scripts/*.service scripts/*.sh dist/paperless-ng/scripts/
cp src/ dist/paperless-ng/src -r
cp docs/_build/html/ dist/paperless-ng/docs -r
-
name: Compile messages
run: |
cd dist/paperless-ng/src
python3 manage.py compilemessages
-
name: Collect static files
run: |
cd dist/paperless-ng/src
python3 manage.py collectstatic --no-input
-
name: Make release package
run: |
cd dist
find . -name __pycache__ | xargs rm -r
tar -cJf paperless-ng.tar.xz paperless-ng/
-
name: Upload release artifact
uses: actions/upload-artifact@v2
with:
name: release
path: dist/paperless-ng.tar.xz
publish-release:
runs-on: ubuntu-latest
needs: build-release
if: contains(github.ref, 'refs/tags/ng-')
steps:
-
name: Download release artifact
uses: actions/download-artifact@v2
with:
name: release
path: ./
-
name: Get branch name
id: get_branch_name
run: |
echo ::set-output name=branch_name::${GITHUB_REF#refs/tags/ng-}
-
name: Create release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.get_branch_name.outputs.branch_name }}
release_name: Paperless-ng ${{ steps.get_branch_name.outputs.branch_name }}
draft: false
prerelease: true
body: |
TODO: Insert changelog here (it's also in the documentation)
-
name: Upload release archive
id: upload-release-asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
asset_path: ./paperless-ng.tar.xz
asset_name: paperless-ng-${{ steps.get_branch_name.outputs.branch_name }}.tar.xz
asset_content_type: application/x-xz
# build and push image to docker hub.
build-docker-image:
if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*')
runs-on: ubuntu-latest
needs: [frontend, tests]
steps:
-
name: Prepare
id: prepare
run: |
VERSION=edge
if [[ $GITHUB_REF == refs/tags/ng-* ]]; then
VERSION=${GITHUB_REF#refs/tags/ng-}
fi
if [[ $GITHUB_REF == refs/heads/* ]]; then
VERSION=${GITHUB_REF#refs/heads/}
fi
echo ::set-output name=version::${VERSION}
-
name: Checkout
uses: actions/checkout@v2
-
name: Download frontend artifact
uses: actions/download-artifact@v2
with:
name: frontend-compiled
path: src/documents/static/frontend/
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
-
name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build and push
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm/v7,linux/arm64
push: true
tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }}
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
-
name: Inspect image
run: |
docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }}

4
.gitignore vendored
View File

@ -65,8 +65,8 @@ target/
.virtualenv
virtualenv
/venv
docker-compose.env
docker-compose.yml
/docker-compose.env
/docker-compose.yml
# Used for development
scripts/import-for-development

51
.travis.yml
View File

@ -1,51 +0,0 @@
language: python
dist: focal
os: linux
jobs:
include:
- name: "Paperless on Python 3.6"
python: "3.6"
- name: "Paperless on Python 3.7"
python: "3.7"
- name: "Paperless on Python 3.8"
python: "3.8"
- name: "Documentation"
script:
- cd docs/
- make html
after_success: true
- name: "Front end"
language: node_js
node_js:
- 15
before_install: true
install:
- cd src-ui/
- npm install -g @angular/cli
- npm install
script:
- ng build --prod
after_success: true
before_install:
- sudo apt-get update -qq
- sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng
install:
- pip install --upgrade pipenv
- pipenv install --system --dev
script:
- cd src/
- pipenv run pytest --cov
- pipenv run pycodestyle
after_success:
- pipenv run coveralls

53
docker/local/Dockerfile → Dockerfile
View File

@ -1,45 +1,62 @@
FROM ubuntu:20.04 AS jbig2enc
WORKDIR /usr/src/jbig2enc
RUN apt-get update && apt-get install -y --no-install-recommends build-essential automake libtool libleptonica-dev zlib1g-dev git ca-certificates
RUN git clone https://github.com/agl/jbig2enc .
RUN ./autogen.sh
RUN ./configure && make
FROM python:3.7-slim
WORKDIR /usr/src/paperless/
COPY requirements.txt ./
#Dependencies
# Binary dependencies
RUN apt-get update \
&& apt-get -y --no-install-recommends install \
build-essential \
# Basic dependencies
curl \
file \
fonts-liberation \
gettext \
ghostscript \
gnupg \
icc-profiles-free \
imagemagick \
libatlas-base-dev \
liblept5 \
libmagic-dev \
libpoppler-cpp-dev \
libpq-dev \
libqpdf-dev \
libxml2 \
libxslt1-dev \
mime-support \
optipng \
sudo \
tzdata \
# OCRmyPDF dependencies
ghostscript \
icc-profiles-free \
liblept5 \
libxml2 \
pngquant \
qpdf \
sudo \
tesseract-ocr \
tesseract-ocr-eng \
tesseract-ocr-deu \
tesseract-ocr-fra \
tesseract-ocr-ita \
tesseract-ocr-spa \
tzdata \
unpaper \
zlib1g \
&& pip3 install --upgrade supervisor setuptools \
&& pip install --no-cache-dir -r requirements.txt \
&& rm -rf /var/lib/apt/lists/*
# Python dependencies
RUN apt-get update \
&& apt-get -y --no-install-recommends install \
build-essential \
libatlas-base-dev \
libmagic-dev \
libpoppler-cpp-dev \
libpq-dev \
libqpdf-dev \
&& python3 -m pip install --upgrade --no-cache-dir supervisor \
&& python3 -m pip install --no-cache-dir -r requirements.txt \
&& apt-get -y purge build-essential libqpdf-dev \
&& apt-get -y autoremove --purge \
&& rm -rf /var/lib/apt/lists/* \
@ -52,6 +69,12 @@ COPY docker/gunicorn.conf.py ./
COPY docker/supervisord.conf /etc/supervisord.conf
COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh
# copy jbig2enc
COPY --from=jbig2enc /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
COPY --from=jbig2enc /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
COPY --from=jbig2enc /usr/src/jbig2enc/src/*.h /usr/local/include/
# copy app
COPY src/ ./src/

6
Pipfile
View File

@ -8,9 +8,6 @@ url = "https://www.piwheels.org/simple"
verify_ssl = true
name = "piwheels"
[requires]
python_version = "3.6"
[packages]
dateparser = "~=0.7.6"
django = "~=3.1.3"
@ -26,7 +23,8 @@ imap-tools = "*"
langdetect = "*"
pdftotext = "*"
pathvalidate = "*"
pillow = "*"
# pinned to 8.1.0, since aarch64 wheels might not be available beyond that
pillow = "==8.1.0"
pikepdf = "*"
python-gnupg = "*"
python-dotenv = "*"

21
Pipfile.lock generated
View File

@ -1,12 +1,10 @@
{
"_meta": {
"hash": {
"sha256": "c35d84fd7f4f1c7d599039712362935e7c41a226b0ab3d83d8c1c2fb2ad0962a"
"sha256": "27c8973d55150371bd18406cc90e61ad4a7ec4b9353007045d611a87120a2703"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.6"
},
"requires": {},
"sources": [
{
"name": "pypi",
@ -439,6 +437,7 @@
"pillow": {
"hashes": [
"sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6",
"sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded",
"sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865",
"sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174",
"sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032",
@ -464,9 +463,12 @@
"sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8",
"sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59",
"sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d",
"sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7",
"sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a",
"sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0",
"sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b",
"sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d"
"sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d",
"sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae"
],
"index": "pypi",
"version": "==8.1.0"
@ -1375,6 +1377,15 @@
"index": "pypi",
"version": "==3.20.1"
},
"typing-extensions": {
"hashes": [
"sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
"sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
"sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
],
"markers": "python_version < '3.8'",
"version": "==3.7.4.3"
},
"urllib3": {
"hashes": [
"sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",

7
compile-frontend.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
set -e
cd src-ui
npm install
./node_modules/.bin/ng build --prod

0
docker/docker-compose.env → docker-compose.example.env
View File

6
docker/local/docker-compose.postgres.yml → docker-compose.postgres.yml
View File

@ -2,11 +2,11 @@ version: "3.4"
services:
broker:
image: redis:6.0
restart: always
restart: unless-stopped
db:
image: postgres:13
restart: always
restart: unless-stopped
volumes:
- pgdata:/var/lib/postgresql/data
environment:
@ -16,7 +16,7 @@ services:
webserver:
build: .
restart: always
restart: unless-stopped
depends_on:
- db
- broker

4
docker/local/docker-compose.sqlite.yml → docker-compose.sqlite.yml
View File

@ -2,11 +2,11 @@ version: "3.4"
services:
broker:
image: redis:6.0
restart: always
restart: unless-stopped
webserver:
build: .
restart: always
restart: unless-stopped
depends_on:
- broker
ports:

4
docker/local/docker-compose.tika.yml → docker-compose.tika.yml
View File

@ -2,11 +2,11 @@ version: "3.4"
services:
broker:
image: redis:6.0
restart: always
restart: unless-stopped
webserver:
build: .
restart: always
restart: unless-stopped
depends_on:
- broker
ports:

1
docker/hub/.env Normal file
View File

@ -0,0 +1 @@
COMPOSE_PROJECT_NAME=paperless

34
docker/hub/docker-compose.env Normal file
View File

@ -0,0 +1,34 @@
# The UID and GID of the user used to run paperless in the container. Set this
# to your UID and GID on the host so that you have write access to the
# consumption directory.
#USERMAP_UID=1000
#USERMAP_GID=1000
# Additional languages to install for text recognition, separated by a
# whitespace. Note that this is
# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the
# default language used when guessing the language from the OCR output.
# The container installs English, German, Italian, Spanish and French by
# default.
# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster
# for available languages.
#PAPERLESS_OCR_LANGUAGES=tur ces
###############################################################################
# Paperless-specific settings #
###############################################################################
# All settings defined in the paperless.conf.example can be used here. The
# Docker setup does not use the configuration file.
# A few commonly adjusted settings are provided below.
# Adjust this key if you plan to make paperless available publicly. It should
# be a very long sequence of random characters. You don't need to remember it.
#PAPERLESS_SECRET_KEY=change-me
# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC.
#PAPERLESS_TIME_ZONE=America/Los_Angeles
# The default language to use for OCR. Set this to the language most of your
# documents are written in.
#PAPERLESS_OCR_LANGUAGE=eng

8
docker/hub/docker-compose.postgres.yml
View File

@ -2,11 +2,11 @@ version: "3.4"
services:
broker:
image: redis:6.0
restart: always
restart: unless-stopped
db:
image: postgres:13
restart: always
restart: unless-stopped
volumes:
- pgdata:/var/lib/postgresql/data
environment:
@ -15,8 +15,8 @@ services:
POSTGRES_PASSWORD: paperless
webserver:
image: jonaswinkler/paperless-ng:0.9.14
restart: always
image: jonaswinkler/paperless-ng:latest
restart: unless-stopped
depends_on:
- db
- broker

6
docker/hub/docker-compose.sqlite.yml
View File

@ -2,11 +2,11 @@ version: "3.4"
services:
broker:
image: redis:6.0
restart: always
restart: unless-stopped
webserver:
image: jonaswinkler/paperless-ng:0.9.14
restart: always
image: jonaswinkler/paperless-ng:latest
restart: unless-stopped
depends_on:
- broker
ports:

6
docker/hub/docker-compose.tika.yml
View File

@ -2,11 +2,11 @@ version: "3.4"
services:
broker:
image: redis:6.0
restart: always
restart: unless-stopped
webserver:
image: jonaswinkler/paperless-ng:0.9.14
restart: always
image: jonaswinkler/paperless-ng:latest
restart: unless-stopped
depends_on:
- broker
ports:

67
docs/administration.rst
View File

@ -48,12 +48,9 @@ Options available to bare-metal and non-docker installations:
Restoring
=========
.. _administration-updating:
Updating paperless
Updating Paperless
##################
If a new release of paperless-ng is available, upgrading depends on how you
@ -69,50 +66,28 @@ First of all, ensure that paperless is stopped.
After that, :ref:`make a backup <administration-backup>`.
A. If you used the dockerfiles archive, simply download the files of the new release,
adjust the settings in the files (i.e., the path to your consumption directory),
and replace your existing docker-compose files. Then start paperless as usual,
which will pull the new image, and update your database, if necessary:
A. If you pull the image from the docker hub, all you need to do is:
.. code:: shell-session
$ cd /path/to/paperless
$ docker-compose pull
$ docker-compose up
If you see everything working, you can start paperless-ng with "-d" to have it
run in the background.
.. hint::
The released docker-compose files specify exact versions to be pulled from the hub.
This is to ensure that if the docker-compose files should change at some point
(i.e., services updates/configured differently), you wont run into trouble due to
docker pulling the ``latest`` image and running it in an older environment.
The docker-compose files refer to the ``latest`` version, which is always the latest
stable release.
B. If you built the image yourself, grab the new archive and replace your current
paperless folder with the new contents.
After that, make the necessary adjustments to the docker-compose.yml (i.e.,
adjust your consumption directory).
Build and start the new image with:
B. If you built the image yourself, do the following:
.. code:: shell-session
$ cd /path/to/paperless
$ git pull
$ ./compile-frontend.sh
$ docker-compose build
$ docker-compose up
If you see everything working, you can start paperless-ng with "-d" to have it
run in the background.
.. hint::
You can usually keep your ``docker-compose.env`` file, since this file will
never include mandatory configuration options. However, it is worth checking
out the new version of this file, since it might have new recommendations
on what to configure.
Running `docker-compose up` will also apply any new database migrations.
If you see everything working, you can start paperless-ng with "-d" to have it
run in the background.
Updating paperless without docker
=================================
@ -134,27 +109,17 @@ After grabbing the new release and unpacking the contents, do the following:
This creates a new virtual environment (or uses your existing environment)
and installs all dependencies into it.
3. Collect static files.
.. code:: shell-session
$ cd src
$ pipenv run python3 manage.py collectstatic --clear
4. Migrate the database.
3. Migrate the database.
.. code:: shell-session
$ cd src
$ pipenv run python3 manage.py migrate
5. Update translation files.
.. code:: shell-session
$ cd src
$ pipenv run python3 manage.py compilemessages
This might not actually do anything. Not every new paperless version comes with new
database migrations.
Management utilities
####################

178
docs/setup.rst
View File

@ -3,35 +3,6 @@
Setup
*****
Download
########
Go to the project page on GitHub and download the
`latest release <https://github.com/jonaswinkler/paperless-ng/releases>`_.
There are multiple options available.
* Download the dockerfiles archive if you want to pull paperless from
Docker Hub.
* Download the dist archive and extract it if you want to build the docker image
yourself or want to install paperless without docker.
.. hint::
In contrast to paperless, the recommended way to get and update paperless-ng
is not to pull the entire git repository. Paperless-ng includes artifacts
that need to be compiled, and that's already done for you in the release.
.. admonition:: Want to try out paperless-ng before migrating?
The release contains a file ``.env`` which sets the docker-compose project
name to "paperless", which is the same as before and instructs docker-compose
to reuse and upgrade your paperless volumes.
Just rename the project name in that file to anything else and docker-compose
will create fresh volumes for you!
Overview of Paperless-ng
########################
@ -110,22 +81,34 @@ Installation
You can go multiple routes with setting up and running Paperless:
* The `docker route`_
* The `bare metal route`_
* :ref:`Pull the image from Docker Hub <setup-docker_hub>`
* :ref:`Build the Docker image yourself <setup-docker_build>`
* :ref:`Install Paperless directly on your system (bare metal) <setup-bare_metal>`
The `docker route`_ is quick & easy. This is the recommended route. This configures all the stuff
The Docker routes are quick & easy. These are the recommended routes. This configures all the stuff
from above automatically so that it just works and uses sensible defaults for all configuration options.
The `bare metal route`_ is more complicated to setup but makes it easier
The bare metal route is more complicated to setup but makes it easier
should you want to contribute some code back. You need to configure and
run the above mentioned components yourself.
.. _setup-docker_route:
.. _setup-docker_hub:
Docker Route
============
Install Paperless from Docker Hub
=================================
1. Install `Docker`_ and `docker-compose`_. [#compose]_
1. Go to the `/docker/hub directory on the project page <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/hub>`_
and download one of the ``docker-compose.*.yml`` files, depending on which database backend you
want to use. Rename this file to `docker-compose.yml`.
Download the ``docker-compose.env`` file and the ``.env`` file as well and store them
in the same directory.
.. hint::
For new installations, it is recommended to use PostgreSQL as the database
backend.
2. Install `Docker`_ and `docker-compose`_.
.. caution::
@ -142,15 +125,7 @@ Docker Route
.. _Docker installation guide: https://docs.docker.com/engine/installation/
.. _docker-compose installation guide: https://docs.docker.com/compose/install/
2. Copy either ``docker-compose.sqlite.yml`` or ``docker-compose.postgres.yml`` to
``docker-compose.yml``, depending on which database backend you want to use.
.. hint::
For new installations, it is recommended to use PostgreSQL as the database
backend.
2. Modify ``docker-compose.yml`` to your preferences. You may want to change the path
3. Modify ``docker-compose.yml`` to your preferences. You may want to change the path
to the consumption directory in this file. Find the line that specifies where
to mount the consumption directory:
@ -167,7 +142,7 @@ Docker Route
Don't change the part after the colon or paperless wont find your documents.
3. Modify ``docker-compose.env``, following the comments in the file. The
4. Modify ``docker-compose.env``, following the comments in the file. The
most important change is to set ``USERMAP_UID`` and ``USERMAP_GID``
to the uid and gid of your user on the host system. This ensures that
both the docker container and you on the host machine have write access
@ -177,9 +152,9 @@ Docker Route
.. note::
You can use any settings from the file ``paperless.conf`` in this file.
You can use any settings from the file ``paperless.conf.example`` in this file.
Have a look at :ref:`configuration` to see whats available.
.. caution::
Certain file systems such as NFS network shares don't support file system
@ -188,11 +163,10 @@ Docker Route
with the default configuration. You will need to use ``PAPERLESS_CONSUMER_POLLING``,
which will disable inotify. See :ref:`here <configuration-polling>`.
4. Run ``docker-compose up -d``. This will create and start the necessary
containers. This will also build the image of paperless if you grabbed the
source archive.
5. Run ``docker-compose up -d``. This will create and start the necessary
containers.
5. To be able to login, you will need a super user. To create it, execute the
6. To be able to login, you will need a super user. To create it, execute the
following command:
.. code-block:: shell-session
@ -202,7 +176,7 @@ Docker Route
This will prompt you to set a username, an optional e-mail address and
finally a password.
6. The default ``docker-compose.yml`` exports the webserver on your local port
7. The default ``docker-compose.yml`` exports the webserver on your local port
8000. If you haven't adapted this, you should now be able to visit your
Paperless instance at ``http://127.0.0.1:8000``. You can login with the
user and password you just created.
@ -210,11 +184,35 @@ Docker Route
.. _Docker: https://www.docker.com/
.. _docker-compose: https://docs.docker.com/compose/install/
.. [#compose] You of course don't have to use docker-compose, but it
simplifies deployment immensely. If you know your way around Docker, feel
free to tinker around without using compose!
.. _setup-docker_build:
.. _`setup-bare_metal`:
Build the docker image yourself
===============================
1. Clone the entire repository of paperless:
.. code:: shell-session
git clone https://github.com/jonaswinkler/paperless-ng
The master branch always reflects the latest stable version.
2. Copy one of the ``docker-compose.*.yml`` to ``docker-compose.yml``,
depending on which database backend you want to use. Copy
``docker-compose.env.example`` to ``docker-compose.env``.
3. Run the `compile-frontend.sh` script. This requires `node` and `npm`.
4. Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run
``docker-compose up -d`` to start the containers, do
.. code:: shell-session
$ docker-compose build
before that to build the image.
.. _setup-bare_metal:
Bare Metal Route
================
@ -229,7 +227,6 @@ writing. Windows is not and will never be supported.
* ``python3-pip``, optionally ``pipenv`` for package installation
* ``python3-dev``
* ``gettext`` for compiling interface translations
* ``fonts-liberation`` for generating thumbnails for plain text files
* ``imagemagick`` >= 6 for PDF conversion
* ``optipng`` for optimizing thumbnails
@ -259,8 +256,9 @@ writing. Windows is not and will never be supported.
3. Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
to use PostgreSQL, SQLite is avialable as well.
4. Get the release archive. If you pull the git repo as it is, you also have to compile the front end by yourself.
Extract the frontend to a place from where you wish to execute it, such as ``/opt/paperless``.
4. Get the release archive from `<https://github.com/jonaswinkler/paperless-ng/releases>`_.
If you clone the git repo as it is, you also have to compile the front end by yourself.
Extract the archive to a place from where you wish to execute it, such as ``/opt/paperless``.
5. Configure paperless. See :ref:`configuration` for details. Edit the included ``paperless.conf`` and adjust the
settings to your needs. Required settings for getting paperless running are:
@ -273,7 +271,7 @@ writing. Windows is not and will never be supported.
paperless stores its data. If you like, you can point both to the same directory.
* ``PAPERLESS_SECRET_KEY`` should be a random sequence of characters. It's used for authentication. Failure
to do so allows third parties to forge authentication credentials.
Many more adjustments can be made to paperless, especially the OCR part. The following options are recommended
for everyone:
@ -282,7 +280,7 @@ writing. Windows is not and will never be supported.
6. Setup permissions. Create a system users under which you wish to run paperless. Ensure that these directories exist
and that the user has write permissions to the following directories
* ``/opt/paperless/media``
* ``/opt/paperless/data``
* ``/opt/paperless/consume``
@ -296,14 +294,8 @@ writing. Windows is not and will never be supported.
.. code:: bash
# This collects static files from paperless and django.
python3 manage.py collectstatic --clear --no-input
# This creates the database schema.
python3 manage.py migrate
# This creates the translation files for paperless.
python3 manage.py compilemessages
# This creates your first paperless user
python3 manage.py createsuperuser
@ -314,7 +306,7 @@ writing. Windows is not and will never be supported.
# This collects static files from paperless and django.
python3 manage.py runserver
and pointing your browser to http://localhost:8000/.
.. warning::
@ -367,13 +359,18 @@ writing. Windows is not and will never be supported.
.. code::
<policy domain="coder" rights="none" pattern="PDF" />
to
.. code::
<policy domain="coder" rights="read|write" pattern="PDF" />
13. Optional: Install the `jbig2enc <https://ocrmypdf.readthedocs.io/en/latest/jbig2.html>`_
encoder. This will reduce the size of generated PDF documents. You'll most likely need
to compile this by yourself, because this software has been patented until around 2017 and
binary packages are not available for most distributions.
Migration to paperless-ng
#########################
@ -407,13 +404,14 @@ Migration to paperless-ng is then performed in a few simple steps:
paperless.
3. Download the latest release of paperless-ng. You can either go with the
docker-compose files or use the archive to build the image yourself.
docker-compose files from `here <https://github.com/jonaswinkler/paperless-ng/tree/master/docker/hub>`_
or clone the repository to build the image yourself.
You can either replace your current paperless folder or put paperless-ng
in a different location.
.. caution::
The release include a ``.env`` file. This will set the
Paperless includes a ``.env`` file. This will set the
project name for docker compose to ``paperless`` so that paperless-ng will
automatically reuse your existing paperless volumes. When you start it, it
will migrate your existing data. After that, your old paperless installation
@ -423,16 +421,10 @@ Migration to paperless-ng is then performed in a few simple steps:
If you want to switch to PostgreSQL, do that after you migrated your existing
SQLite database.
5. Adjust ``docker-compose.yml`` and
``docker-compose.env`` to your needs.
See `docker route`_ for details on which edits are advised.
5. Adjust ``docker-compose.yml`` and ``docker-compose.env`` to your needs.
See :ref:`setup-docker_hub` for details on which edits are advised.
6. Since ``docker-compose`` would just use the the old paperless image, we need to
manually build a new image:
.. code:: shell-session
$ docker-compose build
6. :ref:`Update paperless. <administration-updating>`
7. In order to find your existing documents with the new search feature, you need
to invoke a one-time operation that will create the search index:
@ -440,7 +432,7 @@ Migration to paperless-ng is then performed in a few simple steps:
.. code:: shell-session
$ docker-compose run --rm webserver document_index reindex
This will migrate your database and create the search index. After that,
paperless will take care of maintaining the index by itself.
@ -453,7 +445,7 @@ Migration to paperless-ng is then performed in a few simple steps:
This will run paperless in the background and automatically start it on system boot.
9. Paperless installed a permanent redirect to ``admin/`` in your browser. This
redirect is still in place and prevents access to the new UI. Clear
redirect is still in place and prevents access to the new UI. Clear your
browsing cache in order to fix this.
10. Optionally, follow the instructions below to migrate your existing data to PostgreSQL.
@ -501,9 +493,9 @@ management commands as below.
$ cd /path/to/paperless
$ docker-compose run --rm webserver /bin/bash
This will launch the container and initialize the PostgreSQL database.
b) Without docker, open a shell in your virtual environment, switch to
the ``src`` directory and create the database schema:
@ -513,7 +505,7 @@ management commands as below.
$ pipenv shell
$ cd src
$ python3 manage.py migrate
This will not copy any data yet.
4. Dump your data from SQLite:
@ -521,7 +513,7 @@ management commands as below.
.. code:: shell-session
$ python3 manage.py dumpdata --database=sqlite --exclude=contenttypes --exclude=auth.Permission > data.json
5. Load your data into PostgreSQL:
.. code:: shell-session
@ -572,7 +564,7 @@ as well.
Considerations for less powerful devices
########################################
Paperless runs on Raspberry Pi. However, some things are rather slow on the Pi and
Paperless runs on Raspberry Pi. However, some things are rather slow on the Pi and
configuring some options in paperless can help improve performance immensely:
* Stick with SQLite to save some resources.
@ -594,17 +586,15 @@ configuring some options in paperless can help improve performance immensely:
For details, refer to :ref:`configuration`.
.. note::
Updating the :ref:`automatic matching algorithm <advanced-automatic_matching>`
takes quite a bit of time. However, the update mechanism checks if your
data has changed before doing the heavy lifting. If you experience the
data has changed before doing the heavy lifting. If you experience the
algorithm taking too much cpu time, consider changing the schedule in the
admin interface to daily. You can also manually invoke the task
by changing the date and time of the next run to today/now.
The actual matching of the algorithm is fast and works on Raspberry Pi as
The actual matching of the algorithm is fast and works on Raspberry Pi as
well as on any other device.
.. _redis: https://redis.io/

2
docs/troubleshooting.rst
View File

@ -75,6 +75,6 @@ You might encounter errors such as:
This happens when paperless does not have permission to delete files inside the consumption directory.
Ensure that ``USERMAP_UID`` and ``USERMAP_GID`` are set to the user id and group id you use on the host operating system, if these are
different from ``1000``. See :ref:`setup-docker_route`.
different from ``1000``. See :ref:`setup-docker_hub`.
Also ensure that you are able to read and write to the consumption directory on the host.

2
docs/usage_overview.rst
View File

@ -118,7 +118,7 @@ to share any documents with paperless. This can be combined with any of the mobi
scanning apps out there, such as Office Lens.
Furthermore, there is the `Paperless App <https://github.com/bauerj/paperless_app>`_ as well,
which no only has document upload, but also document editing and browsing.
which not only has document upload, but also document browsing and download features.
.. _usage-email:

74
requirements.txt Normal file
View File

@ -0,0 +1,74 @@
#
# These requirements were autogenerated by pipenv
# To regenerate from the project's Pipfile, run:
#
# pipenv lock --requirements
#
-i https://pypi.python.org/simple
--extra-index-url https://www.piwheels.org/simple
arrow==0.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
asgiref==3.3.1; python_version >= '3.5'
blessed==1.17.12
certifi==2020.12.5
cffi==1.14.4
chardet==4.0.0; python_version >= '3.1'
coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
dateparser==0.7.6
django-cors-headers==3.6.0
django-extensions==3.1.0
django-filter==2.4.0
django-picklefield==3.0.1; python_version >= '3'
django-q==1.3.4
django==3.1.5
djangorestframework==3.12.2
filelock==3.0.12
fuzzywuzzy==0.18.0
gunicorn==20.0.4
humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
imap-tools==0.34.0
img2pdf==0.4.0
importlib-metadata==3.3.0; python_version < '3.8'
inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
inotifyrecursive==0.3.5
joblib==1.0.0; python_version >= '3.6'
langdetect==1.0.8
lxml==4.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
numpy==1.19.5; python_version >= '3.6'
ocrmypdf==11.4.5
pathvalidate==2.3.2
pdfminer.six==20201018; python_version >= '3.4'
pdftotext==2.1.5
pikepdf==2.2.5
pillow==8.1.0
pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
psycopg2-binary==2.8.6
pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
python-dateutil==2.8.1
python-dotenv==0.15.0
python-gnupg==0.4.6
python-levenshtein==0.12.0
python-magic==0.4.18
pytz==2020.5
redis==3.5.3
regex==2020.11.13
reportlab==3.5.59
requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
scikit-learn==0.23.2
scipy==1.5.4; python_version >= '3.6'
six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
sortedcontainers==2.3.0
sqlparse==0.4.1; python_version >= '3.5'
threadpoolctl==2.1.0; python_version >= '3.5'
tika==1.24
tqdm==4.55.1
typing-extensions==3.7.4.3; python_version < '3.8'
tzlocal==2.1
urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
watchdog==1.0.2
wcwidth==0.2.5
whitenoise==5.2.0
whoosh==2.7.4
zipp==3.4.0; python_version >= '3.6'

125
scripts/make-release.sh
View File

@ -1,125 +0,0 @@
#!/bin/bash
# Release checklist
# - wait for travis build.
# adjust src/paperless/version.py
# changelog in the documentation
# adjust versions in docker/hub/*
# adjust version in src-ui/src/environments/prod
# If docker-compose was modified: all compose files are the same.
# Steps:
# run release script "dev", push
# if it works: new tag, merge into master
# on master: make release "lastest", push
# on master: make release "version-tag", push
# publish release files
set -e
VERSION=$1
if [ -z "$VERSION" ]
then
echo "Need a version string."
exit 1
fi
# source root directory of paperless
PAPERLESS_ROOT=$(git rev-parse --show-toplevel)
# output directory
PAPERLESS_DIST="$PAPERLESS_ROOT/dist"
PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng"
PAPERLESS_DIST_DOCKERFILES="$PAPERLESS_DIST/paperless-ng-dockerfiles"
if [ -d "$PAPERLESS_DIST" ]
then
echo "Removing $PAPERLESS_DIST"
rm "$PAPERLESS_DIST" -r
fi
mkdir "$PAPERLESS_DIST"
mkdir "$PAPERLESS_DIST_APP"
mkdir "$PAPERLESS_DIST_APP/docker"
mkdir "$PAPERLESS_DIST_APP/scripts"
mkdir "$PAPERLESS_DIST_DOCKERFILES"
# setup dependencies.
cd "$PAPERLESS_ROOT"
pipenv clean
pipenv install --dev
pipenv lock --keep-outdated -r > "$PAPERLESS_DIST_APP/requirements.txt"
# test if the application works.
cd "$PAPERLESS_ROOT/src"
#pipenv run pytest --cov
#pipenv run pycodestyle
# make the documentation.
cd "$PAPERLESS_ROOT/docs"
make clean html
# copy stuff into place
# the application itself
cp "$PAPERLESS_ROOT/.env" \
"$PAPERLESS_ROOT/.dockerignore" \
"$PAPERLESS_ROOT/CONTRIBUTING.md" \
"$PAPERLESS_ROOT/LICENSE" \
"$PAPERLESS_ROOT/Pipfile" \
"$PAPERLESS_ROOT/Pipfile.lock" \
"$PAPERLESS_ROOT/README.md" "$PAPERLESS_DIST_APP"
cp "$PAPERLESS_ROOT/paperless.conf.example" "$PAPERLESS_DIST_APP/paperless.conf"
# copy python source, templates and static files.
cd "$PAPERLESS_ROOT"
find src -name '*.po' -o -wholename '*/templates/*' -o -wholename '*/static/*' -o -name '*.py' | cpio -pdm "$PAPERLESS_DIST_APP"
# build the front end.
cd "$PAPERLESS_ROOT/src-ui"
ng build --prod --output-hashing none --sourceMap=false --output-path "$PAPERLESS_DIST_APP/src/documents/static/frontend"
# documentation
cp "$PAPERLESS_ROOT/docs/_build/html/" "$PAPERLESS_DIST_APP/docs" -r
# docker files for building the image yourself
cp "$PAPERLESS_ROOT/docker/local/"* "$PAPERLESS_DIST_APP"
cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_APP"
# docker files for pulling from docker hub
cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST_DOCKERFILES"
cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST_DOCKERFILES"
cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_DOCKERFILES"
# auxiliary files required for the docker image
cp "$PAPERLESS_ROOT/docker/docker-entrypoint.sh" "$PAPERLESS_DIST_APP/docker/"
cp "$PAPERLESS_ROOT/docker/gunicorn.conf.py" "$PAPERLESS_DIST_APP/docker/"
cp "$PAPERLESS_ROOT/docker/imagemagick-policy.xml" "$PAPERLESS_DIST_APP/docker/"
cp "$PAPERLESS_ROOT/docker/supervisord.conf" "$PAPERLESS_DIST_APP/docker/"
# auxiliary files for bare metal installs
cp "$PAPERLESS_ROOT/scripts/paperless-webserver.service" "$PAPERLESS_DIST_APP/scripts/"
cp "$PAPERLESS_ROOT/scripts/paperless-consumer.service" "$PAPERLESS_DIST_APP/scripts/"
cp "$PAPERLESS_ROOT/scripts/paperless-scheduler.service" "$PAPERLESS_DIST_APP/scripts/"
# try to make the docker build.
cd "$PAPERLESS_DIST_APP"
docker build . -t "jonaswinkler/paperless-ng:$VERSION"
# works. package the app!
cd "$PAPERLESS_DIST"
tar -cJf "paperless-ng-$VERSION.tar.xz" paperless-ng/
tar -cJf "paperless-ng-$VERSION-dockerfiles.tar.xz" paperless-ng-dockerfiles/

23
scripts/push-release.sh
View File

@ -1,23 +0,0 @@
#!/bin/bash
set -e
VERSION=$1
if [ -z "$VERSION" ]
then
echo "Need a version string."
exit 1
fi
# source root directory of paperless
PAPERLESS_ROOT=$(git rev-parse --show-toplevel)
# output directory
PAPERLESS_DIST="$PAPERLESS_ROOT/dist"
PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng"
cd "$PAPERLESS_DIST_APP"
docker push "jonaswinkler/paperless-ng:$VERSION"

1
src-ui/angular.json
View File

@ -58,6 +58,7 @@
"with": "src/environments/environment.prod.ts"
}
],
"outputPath": "../src/documents/static/frontend/",
"optimization": true,
"outputHashing": "none",
"sourceMap": false,

2
src/paperless_mail/tests/test_mail.py
View File

@ -41,7 +41,7 @@ class BogusMailBox(ContextManager):
folder = BogusFolderManager()
def fetch(self, criteria, mark_seen):
def fetch(self, criteria, mark_seen, charset=""):
msg = self.messages
criteria = str(criteria).strip('()').split(" ")