paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-02 13:45:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

rebase mariadb feature onto dev

Browse Source
This commit is contained in:
Brian Kelly 2022-05-03 20:13:13 -06:00 committed by Trenton Holmes
parent 15d074d39c
commit 65cfd55027
10 changed files with 268 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -56,6 +56,7 @@ ARG RUNTIME_PACKAGES="\
curl \
file \
# fonts for text file thumbnail generation
default-libmysqlclient-dev \
fonts-liberation \
gettext \
ghostscript \
@ -186,6 +187,7 @@ COPY Pipfile* ./
ARG BUILD_PACKAGES="\
build-essential \
git \
mariadb-client \
python3-dev"
RUN set -eux \

101
docker/compose/docker-compose.mariadb-tika.yml Normal file
View File

@ -0,0 +1,101 @@
# docker-compose file for running paperless from the Docker Hub.
# This file contains everything paperless needs to run.
# Paperless supports amd64, arm and arm64 hardware.
#
# All compose files of paperless configure paperless in the following way:
#
# - Paperless is (re)started on system boot, if it was running before shutdown.
# - Docker volumes for storing data are managed by Docker.
# - Folders for importing and exporting files are created in the same directory
# as this file and mounted to the correct folders inside the container.
# - Paperless listens on port 8000.
#
# In addition to that, this docker-compose file adds the following optional
# configurations:
#
# - Instead of SQLite (default), MariaDB is used as the database server.
# - Apache Tika and Gotenberg servers are started with paperless and paperless
# is configured to use these services. These provide support for consuming
# Office documents (Word, Excel, Power Point and their LibreOffice counter-
# parts.
#
# To install and update paperless with this file, do the following:
#
# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
# and '.env' into a folder.
# - Run 'docker-compose pull'.
# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
# - Run 'docker-compose up -d'.
#
# For more extensive installation and update instructions, refer to the
# documentation.
version: "3.4"
services:
broker:
image: redis:6.0
restart: unless-stopped
volumes:
- redisdata:/data
db:
image: mariadb:10
restart: unless-stopped
volumes:
- dbdata:/var/lib/mariadb/data
environment:
MARIADB_HOST: paperless
MARIADB_DATABASE: paperless
MARIADB_USER: paperless
MARIADB_PASSWORD: paperless
MARIADB_ROOT_PASSWORD: paperless
ports:
- "3306:3306"
webserver:
image: ghcr.io/paperless-ngx/paperless-ngx:latest
restart: unless-stopped
depends_on:
- db
- broker
- gotenberg
- tika
ports:
- 8000:8000
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000"]
interval: 30s
timeout: 10s
retries: 5
volumes:
- data:/usr/src/paperless/data
- media:/usr/src/paperless/media
- ./export:/usr/src/paperless/export
- ./consume:/usr/src/paperless/consume
env_file: docker-compose.env
environment:
PAPERLESS_REDIS: redis://broker:6379
PAPERLESS_DBENGINE: mariadb
PAPERLESS_DBHOST: db
PAPERLESS_DBUSER: root
PAPERLESS_DBPASSWORD: paperless
PAPERLESS_DBPORT: 3306
PAPERLESS_TIKA_ENABLED: 1
PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
gotenberg:
image: gotenberg/gotenberg:7
restart: unless-stopped
environment:
CHROMIUM_DISABLE_ROUTES: 1
tika:
image: apache/tika
restart: unless-stopped
volumes:
data:
media:
dbdata:
redisdata:

84
docker/compose/docker-compose.mariadb.yml Normal file
View File

@ -0,0 +1,84 @@
# docker-compose file for running paperless from the Docker Hub.
# This file contains everything paperless needs to run.
# Paperless supports amd64, arm and arm64 hardware.
#
# All compose files of paperless configure paperless in the following way:
#
# - Paperless is (re)started on system boot, if it was running before shutdown.
# - Docker volumes for storing data are managed by Docker.
# - Folders for importing and exporting files are created in the same directory
# as this file and mounted to the correct folders inside the container.
# - Paperless listens on port 8000.
#
# In addition to that, this docker-compose file adds the following optional
# configurations:
#
# - Instead of SQLite (default), MariaDB is used as the database server.
#
# To install and update paperless with this file, do the following:
#
# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
# and '.env' into a folder.
# - Run 'docker-compose pull'.
# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
# - Run 'docker-compose up -d'.
#
# For more extensive installation and update instructions, refer to the
# documentation.
version: "3.4"
services:
broker:
image: redis:6.0
restart: unless-stopped
volumes:
- redisdata:/data
db:
image: mariadb:10
restart: unless-stopped
volumes:
- dbdata:/var/lib/mariadb/data
environment:
MARIADB_HOST: paperless
MARIADB_DATABASE: paperless
MARIADB_USER: paperless
MARIADB_PASSWORD: paperless
MARIADB_ROOT_PASSWORD: paperless
ports:
- "3306:3306"
webserver:
# image: ghcr.io/paperless-ngx/paperless-ngx:latest
image: paperless-bktest:latest
restart: unless-stopped
depends_on:
- db
- broker
ports:
- 8000:8000
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000"]
interval: 30s
timeout: 10s
retries: 5
volumes:
- data:/usr/src/paperless/data
- media:/usr/src/paperless/media
- ./export:/usr/src/paperless/export
- ./consume:/usr/src/paperless/consume
env_file: docker-compose.env
environment:
PAPERLESS_REDIS: redis://broker:6379
PAPERLESS_DBENGINE: mariadb
PAPERLESS_DBHOST: db
PAPERLESS_DBUSER: root
PAPERLESS_DBPASSWORD: paperless
PAPERLESS_DBPORT: 3306
volumes:
data:
media:
dbdata:
redisdata:

28
docker/docker-prepare.sh
View File

@ -28,6 +28,30 @@ wait_for_postgres() {
done
}
wait_for_mariadb() {
echo "Waiting for MariaDB to start..."
host="${PAPERLESS_DBHOST:=localhost}"
port="${PAPERLESS_DBPORT:=3306}"
attempt_num=1
max_attempts=5
while ! </dev/tcp/$host/$port; do
if [ $attempt_num -eq $max_attempts ]; then
echo "Unable to connect to database."
exit 1
else
echo "Attempt $attempt_num failed! Trying again in 5 seconds..."
fi
attempt_num=$(expr "$attempt_num" + 1)
sleep 5
done
}
wait_for_redis() {
# We use a Python script to send the Redis ping
# instead of installing redis-tools just for 1 thing
@ -66,7 +90,9 @@ superuser() {
}
do_work() {
if [[ -n "${PAPERLESS_DBHOST}" ]]; then
if [[ "${PAPERLESS_DBENGINE}" == "mariadb" ]]; then
wait_for_mariadb
elif [[ -n "${PAPERLESS_DBHOST}" ]]; then
wait_for_postgres
fi

6
docs/administration.rst
View File

@ -35,13 +35,15 @@ Options available to docker installations:
``/var/lib/docker/volumes`` on the host and you need to be root in order
to access them.
Paperless uses 3 volumes:
Paperless uses 4 volumes:
* ``paperless_media``: This is where your documents are stored.
* ``paperless_data``: This is where auxillary data is stored. This
folder also contains the SQLite database, if you use it.
* ``paperless_pgdata``: Exists only if you use PostgreSQL and contains
the database.
* ``paperless_dbdata``: Exists only if you use MariaDB and contains
the database.
Options available to bare-metal and non-docker installations:
@ -49,7 +51,7 @@ Options available to bare-metal and non-docker installations:
crashes at some point or your disk fails, you can simply copy the folder back
into place and it works.
When using PostgreSQL, you'll also have to backup the database.
When using PostgreSQL or MariaDB, you'll also have to backup the database.
.. _migrating-restoring:

14
docs/configuration.rst
View File

@ -29,9 +29,15 @@ PAPERLESS_REDIS=<url>
Defaults to redis://localhost:6379.
PAPERLESS_DBENGINE=<engine_name>
Optional, gives the ability to choose Postgres or MariaDB for database engine.
Available options are `postgresql` and `mariadb`.
Default is `postgresql`.
PAPERLESS_DBHOST=<hostname>
By default, sqlite is used as the database backend. This can be changed here.
Set PAPERLESS_DBHOST and PostgreSQL will be used instead of sqlite.
Set PAPERLESS_DBHOST and another database will be used instead of sqlite.
PAPERLESS_DBPORT=<port>
Adjust port if necessary.
@ -39,17 +45,17 @@ PAPERLESS_DBPORT=<port>
Default is 5432.
PAPERLESS_DBNAME=<name>
Database name in PostgreSQL.
Database name in PostgreSQL or MariaDB.
Defaults to "paperless".
PAPERLESS_DBUSER=<name>
Database user in PostgreSQL.
Database user in PostgreSQL or MariaDB.
Defaults to "paperless".
PAPERLESS_DBPASS=<password>
Database password for PostgreSQL.
Database password for PostgreSQL or MariaDB.
Defaults to "paperless".

2
docs/extending.rst
View File

@ -79,7 +79,7 @@ To do the setup you need to perform the steps from the following chapters in a c
6. You can now either ...
* install redis or
* use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a postgresql server) or
* use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a database server) or
* spin up a bare redis container
.. code:: shell-session

7
docs/setup.rst
View File

@ -73,7 +73,7 @@ Paperless consists of the following components:
for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different
process (maybe even on different machines!), and therefore, this is necessary.
* Optional: A database server. Paperless supports both PostgreSQL and SQLite for storing its data.
* Optional: A database server. Paperless supports PostgreSQL, MariaDB and SQLite for storing its data.
Installation
@ -316,11 +316,13 @@ writing. Windows is not and will never be supported.
* ``python3-pip``
* ``python3-dev``
* ``default-libmysqlclient-dev`` for MariaDB
* ``fonts-liberation`` for generating thumbnails for plain text files
* ``imagemagick`` >= 6 for PDF conversion
* ``gnupg`` for handling encrypted documents
* ``libpq-dev`` for PostgreSQL
* ``libmagic-dev`` for mime type detection
* ``mariadb-client`` for MariaDB compile time
* ``mime-support`` for mime type detection
* ``libzbar0`` for barcode detection
* ``poppler-utils`` for barcode detection
@ -361,7 +363,7 @@ writing. Windows is not and will never be supported.
2. Install ``redis`` >= 5.0 and configure it to start automatically.
3. Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
to use PostgreSQL, SQLite is available as well.
to use PostgreSQL, MariaDB and SQLite are available as well.
.. note::
@ -377,6 +379,7 @@ writing. Windows is not and will never be supported.
settings to your needs. Required settings for getting paperless running are:
* ``PAPERLESS_REDIS`` should point to your redis server, such as redis://localhost:6379.
* ``PAPERLESS_DBENGINE`` optional, and should be one of `postgres, mariadb, or sqlite`
* ``PAPERLESS_DBHOST`` should be the hostname on which your PostgreSQL server is running. Do not configure this
to use SQLite instead. Also configure port, database name, user and password as necessary.
* ``PAPERLESS_CONSUMPTION_DIR`` should point to a folder which paperless should watch for documents. You might

23
install-paperless-ngx.sh
View File

@ -118,12 +118,12 @@ ask "Current time zone" "$default_time_zone"
TIME_ZONE=$ask_result
echo ""
echo "Database backend: PostgreSQL and SQLite are available. Use PostgreSQL"
echo "Database backend: PostgreSQL, MariaDB, and SQLite are available. Use PostgreSQL"
echo "if unsure. If you're running on a low-power device such as Raspberry"
echo "Pi, use SQLite to save resources."
echo ""
ask "Database backend" "postgres" "postgres sqlite"
ask "Database backend" "postgres" "postgres sqlite" "mariadb"
DATABASE_BACKEND=$ask_result
echo ""
@ -227,6 +227,18 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
POSTGRES_FOLDER=$ask_result
fi
if [[ "$DATABASE_BACKEND" == "mariadb" ]] ; then
echo ""
echo "The database folder, where mariadb stores its data."
echo ""
echo "CAUTION: If specified, you must specify an absolute path starting with /"
echo "or a relative path starting with ./ here."
echo ""
ask_docker_folder "Database folder" ""
MARIADB_FOLDER=$ask_result
fi
echo ""
echo "3. Login credentials"
echo "===================="
@ -285,6 +297,9 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
echo "Database (postgres) folder: $POSTGRES_FOLDER"
fi
fi
if [[ "$DATABASE_BACKEND" == "mariadb" ]] ; then
echo "Database (mariadb) folder: $MARIADB_FOLDER"
fi
echo ""
echo "URL: $URL"
echo "Port: $PORT"
@ -361,6 +376,10 @@ if [[ -n $POSTGRES_FOLDER ]] ; then
sed -i "/^\s*pgdata:/d" docker-compose.yml
fi
if [[ -n $MARIADB_FOLDER ]] ; then
sed -i "s#- dbdata:/var/lib/mariadb/data#- $MARIADB_FOLDER:/var/lib/mariadb/data#g" docker-compose.yml
fi
# remove trailing blank lines from end of file
sed -i -e :a -e '/^\n*$/{$d;N;};/\n$/ba' docker-compose.yml
# if last line in file contains "volumes:", remove that line since no more named volumes are left

15
src/paperless/settings.py
View File

@ -328,16 +328,27 @@ if os.getenv("PAPERLESS_DBHOST"):
DATABASES["sqlite"] = DATABASES["default"].copy()
DATABASES["default"] = {
"ENGINE": "django.db.backends.postgresql_psycopg2",
"HOST": os.getenv("PAPERLESS_DBHOST"),
"NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
"USER": os.getenv("PAPERLESS_DBUSER", "paperless"),
"PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"),
"OPTIONS": {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")},
"OPTIONS": {},
}
if os.getenv("PAPERLESS_DBPORT"):
DATABASES["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT")
# Leave room for future extensibility
if os.getenv("PAPERLESS_DBENGINE") == "mariadb":
engine = "django.db.backends.mysql"
options = {"read_default_file": "/etc/mysql/my.cnf"}
else: # Default to PostgresDB
engine = "django.db.backends.postgresql_psycopg2"
options = {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")}
DATABASES["default"]["ENGINE"] = engine
for key, value in options.items():
DATABASES["default"]["OPTIONS"][key] = value
if os.getenv("PAPERLESS_DB_TIMEOUT") is not None:
_new_opts = {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}
if "OPTIONS" in DATABASES["default"]: