rebase mariadb feature onto dev

2025-10-30 03:56:23 -05:00 · 2022-05-03 20:13:13 -06:00
parent 15d074d39c
commit 65cfd55027
10 changed files with 268 additions and 14 deletions
--- a/2
+++ b/2
@@ -56,6 +56,7 @@ ARG RUNTIME_PACKAGES="\
  curl \
  file \
  # fonts for text file thumbnail generation
+  default-libmysqlclient-dev \
  fonts-liberation \
  gettext \
  ghostscript \
@@ -186,6 +187,7 @@ COPY Pipfile* ./
 ARG BUILD_PACKAGES="\
  build-essential \
  git \
+  mariadb-client \
  python3-dev"

 RUN set -eux \
--- a/docker/compose/docker-compose.mariadb-tika.yml
+++ b/docker/compose/docker-compose.mariadb-tika.yml
@@ -0,0 +1,101 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Instead of SQLite (default), MariaDB is used as the database server.
+# - Apache Tika and Gotenberg servers are started with paperless and paperless
+#   is configured to use these services. These provide support for consuming
+#   Office documents (Word, Excel, Power Point and their LibreOffice counter-
+#   parts.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: redis:6.0
+    restart: unless-stopped
+    volumes:
+      - redisdata:/data
+
+  db:
+    image: mariadb:10
+    restart: unless-stopped
+    volumes:
+      - dbdata:/var/lib/mariadb/data
+    environment:
+      MARIADB_HOST: paperless
+      MARIADB_DATABASE: paperless
+      MARIADB_USER: paperless
+      MARIADB_PASSWORD: paperless
+      MARIADB_ROOT_PASSWORD: paperless
+    ports:
+      - "3306:3306"
+
+  webserver:
+    image: ghcr.io/paperless-ngx/paperless-ngx:latest
+    restart: unless-stopped
+    depends_on:
+      - db
+      - broker
+      - gotenberg
+      - tika
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_DBENGINE: mariadb
+      PAPERLESS_DBHOST: db
+      PAPERLESS_DBUSER: root
+      PAPERLESS_DBPASSWORD: paperless
+      PAPERLESS_DBPORT: 3306
+      PAPERLESS_TIKA_ENABLED: 1
+      PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
+      PAPERLESS_TIKA_ENDPOINT: http://tika:9998
+
+  gotenberg:
+    image: gotenberg/gotenberg:7
+    restart: unless-stopped
+    environment:
+      CHROMIUM_DISABLE_ROUTES: 1
+
+  tika:
+    image: apache/tika
+    restart: unless-stopped
+
+volumes:
+  data:
+  media:
+  dbdata:
+  redisdata:
--- a/docker/compose/docker-compose.mariadb.yml
+++ b/docker/compose/docker-compose.mariadb.yml
@@ -0,0 +1,84 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Instead of SQLite (default), MariaDB is used as the database server.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: redis:6.0
+    restart: unless-stopped
+    volumes:
+      - redisdata:/data
+
+  db:
+    image: mariadb:10
+    restart: unless-stopped
+    volumes:
+      - dbdata:/var/lib/mariadb/data
+    environment:
+      MARIADB_HOST: paperless
+      MARIADB_DATABASE: paperless
+      MARIADB_USER: paperless
+      MARIADB_PASSWORD: paperless
+      MARIADB_ROOT_PASSWORD: paperless
+    ports:
+      - "3306:3306"
+
+  webserver:
+#    image: ghcr.io/paperless-ngx/paperless-ngx:latest
+    image: paperless-bktest:latest
+    restart: unless-stopped
+    depends_on:
+      - db
+      - broker
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_DBENGINE: mariadb
+      PAPERLESS_DBHOST: db
+      PAPERLESS_DBUSER: root
+      PAPERLESS_DBPASSWORD: paperless
+      PAPERLESS_DBPORT: 3306
+
+
+volumes:
+  data:
+  media:
+  dbdata:
+  redisdata:
--- a/docker/docker-prepare.sh
+++ b/docker/docker-prepare.sh
@@ -28,6 +28,30 @@ wait_for_postgres() {
 	done
 }

+wait_for_mariadb() {
+	echo "Waiting for MariaDB to start..."
+
+	host="${PAPERLESS_DBHOST:=localhost}"
+	port="${PAPERLESS_DBPORT:=3306}"
+
+	attempt_num=1
+	max_attempts=5
+
+	while ! </dev/tcp/$host/$port; do
+
+		if [ $attempt_num -eq $max_attempts ]; then
+			echo "Unable to connect to database."
+			exit 1
+		else
+			echo "Attempt $attempt_num failed! Trying again in 5 seconds..."
+
+		fi
+
+		attempt_num=$(expr "$attempt_num" + 1)
+		sleep 5
+	done
+}
+
 wait_for_redis() {
 	# We use a Python script to send the Redis ping
 	# instead of installing redis-tools just for 1 thing
@@ -66,7 +90,9 @@ superuser() {
 }

 do_work() {
-	if [[ -n "${PAPERLESS_DBHOST}" ]]; then
+	if [[ "${PAPERLESS_DBENGINE}" == "mariadb" ]]; then
+		wait_for_mariadb
+	elif [[ -n "${PAPERLESS_DBHOST}" ]]; then
 		wait_for_postgres
 	fi

--- a/docs/administration.rst
+++ b/docs/administration.rst
@@ -35,13 +35,15 @@ Options available to docker installations:
    ``/var/lib/docker/volumes`` on the host and you need to be root in order
    to access them.

-    Paperless uses 3 volumes:
+    Paperless uses 4 volumes:

    *   ``paperless_media``: This is where your documents are stored.
    *   ``paperless_data``: This is where auxillary data is stored. This
        folder also contains the SQLite database, if you use it.
    *   ``paperless_pgdata``: Exists only if you use PostgreSQL and contains
        the database.
+    *   ``paperless_dbdata``: Exists only if you use MariaDB and contains
+        the database.

 Options available to bare-metal and non-docker installations:

@@ -49,7 +51,7 @@ Options available to bare-metal and non-docker installations:
    crashes at some point or your disk fails, you can simply copy the folder back
    into place and it works.

-    When using PostgreSQL, you'll also have to backup the database.
+    When using PostgreSQL or MariaDB, you'll also have to backup the database.

 .. _migrating-restoring:

--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -29,9 +29,15 @@ PAPERLESS_REDIS=<url>

    Defaults to redis://localhost:6379.

+PAPERLESS_DBENGINE=<engine_name>
+    Optional, gives the ability to choose Postgres or MariaDB for database engine.
+    Available options are `postgresql` and `mariadb`.
+    Default is `postgresql`.
+
 PAPERLESS_DBHOST=<hostname>
    By default, sqlite is used as the database backend. This can be changed here.
-    Set PAPERLESS_DBHOST and PostgreSQL will be used instead of sqlite.
+
+    Set PAPERLESS_DBHOST and another database will be used instead of sqlite.

 PAPERLESS_DBPORT=<port>
    Adjust port if necessary.
@@ -39,17 +45,17 @@ PAPERLESS_DBPORT=<port>
    Default is 5432.

 PAPERLESS_DBNAME=<name>
-    Database name in PostgreSQL.
+    Database name in PostgreSQL or MariaDB.

    Defaults to "paperless".

 PAPERLESS_DBUSER=<name>
-    Database user in PostgreSQL.
+    Database user in PostgreSQL or MariaDB.

    Defaults to "paperless".

 PAPERLESS_DBPASS=<password>
-    Database password for PostgreSQL.
+    Database password for PostgreSQL or MariaDB.

    Defaults to "paperless".

--- a/docs/extending.rst
+++ b/docs/extending.rst
@@ -79,7 +79,7 @@ To do the setup you need to perform the steps from the following chapters in a c
 6.  You can now either ...

    *  install redis or
-    *  use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a postgresql server) or
+    *  use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a database server) or
    *  spin up a bare redis container

        .. code:: shell-session
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -73,7 +73,7 @@ Paperless consists of the following components:
    for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different
    process (maybe even on different machines!), and therefore, this is necessary.

-*   Optional: A database server. Paperless supports both PostgreSQL and SQLite for storing its data.
+*   Optional: A database server. Paperless supports PostgreSQL, MariaDB and SQLite for storing its data.


 Installation
@@ -316,11 +316,13 @@ writing. Windows is not and will never be supported.
    *   ``python3-pip``
    *   ``python3-dev``

+    *   ``default-libmysqlclient-dev`` for MariaDB
    *   ``fonts-liberation`` for generating thumbnails for plain text files
    *   ``imagemagick`` >= 6 for PDF conversion
    *   ``gnupg`` for handling encrypted documents
    *   ``libpq-dev`` for PostgreSQL
    *   ``libmagic-dev`` for mime type detection
+    *   ``mariadb-client`` for MariaDB compile time
    *   ``mime-support`` for mime type detection
    *   ``libzbar0`` for barcode detection
    *   ``poppler-utils`` for barcode detection
@@ -361,7 +363,7 @@ writing. Windows is not and will never be supported.
 2.  Install ``redis`` >= 5.0 and configure it to start automatically.

 3.  Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
-    to use PostgreSQL, SQLite is available as well.
+    to use PostgreSQL, MariaDB and SQLite are available as well.

    .. note::

@@ -377,6 +379,7 @@ writing. Windows is not and will never be supported.
    settings to your needs. Required settings for getting paperless running are:

    *   ``PAPERLESS_REDIS`` should point to your redis server, such as redis://localhost:6379.
+    *   ``PAPERLESS_DBENGINE`` optional, and should be one of `postgres, mariadb, or sqlite`
    *   ``PAPERLESS_DBHOST`` should be the hostname on which your PostgreSQL server is running. Do not configure this
        to use SQLite instead. Also configure port, database name, user and password as necessary.
    *   ``PAPERLESS_CONSUMPTION_DIR`` should point to a folder which paperless should watch for documents. You might
--- a/install-paperless-ngx.sh
+++ b/install-paperless-ngx.sh
@@ -118,12 +118,12 @@ ask "Current time zone" "$default_time_zone"
 TIME_ZONE=$ask_result

 echo ""
-echo "Database backend: PostgreSQL and SQLite are available. Use PostgreSQL"
+echo "Database backend: PostgreSQL, MariaDB, and SQLite are available. Use PostgreSQL"
 echo "if unsure. If you're running on a low-power device such as Raspberry"
 echo "Pi, use SQLite to save resources."
 echo ""

-ask "Database backend" "postgres" "postgres sqlite"
+ask "Database backend" "postgres" "postgres sqlite" "mariadb"
 DATABASE_BACKEND=$ask_result

 echo ""
@@ -227,6 +227,18 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
 	POSTGRES_FOLDER=$ask_result
 fi

+if [[ "$DATABASE_BACKEND" == "mariadb" ]] ; then
+	echo ""
+	echo "The database folder, where mariadb stores its data."
+	echo ""
+	echo "CAUTION: If specified, you must specify an absolute path starting with /"
+	echo "or a relative path starting with ./ here."
+	echo ""
+
+	ask_docker_folder "Database folder" ""
+	MARIADB_FOLDER=$ask_result
+fi
+
 echo ""
 echo "3. Login credentials"
 echo "===================="
@@ -285,6 +297,9 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
 		echo "Database (postgres) folder: $POSTGRES_FOLDER"
 	fi
 fi
+if [[ "$DATABASE_BACKEND" == "mariadb" ]] ; then
+	echo "Database (mariadb) folder: $MARIADB_FOLDER"
+fi
 echo ""
 echo "URL: $URL"
 echo "Port: $PORT"
@@ -361,6 +376,10 @@ if [[ -n $POSTGRES_FOLDER ]] ; then
 	sed -i "/^\s*pgdata:/d" docker-compose.yml
 fi

+if [[ -n $MARIADB_FOLDER ]] ; then
+	sed -i "s#- dbdata:/var/lib/mariadb/data#- $MARIADB_FOLDER:/var/lib/mariadb/data#g" docker-compose.yml
+fi
+
 # remove trailing blank lines from end of file
 sed -i -e :a -e '/^\n*$/{$d;N;};/\n$/ba' docker-compose.yml
 # if last line in file contains "volumes:", remove that line since no more named volumes are left
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -328,16 +328,27 @@ if os.getenv("PAPERLESS_DBHOST"):
    DATABASES["sqlite"] = DATABASES["default"].copy()

    DATABASES["default"] = {
-        "ENGINE": "django.db.backends.postgresql_psycopg2",
        "HOST": os.getenv("PAPERLESS_DBHOST"),
        "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
        "USER": os.getenv("PAPERLESS_DBUSER", "paperless"),
        "PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"),
-        "OPTIONS": {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")},
+        "OPTIONS": {},
    }
    if os.getenv("PAPERLESS_DBPORT"):
        DATABASES["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT")

+    # Leave room for future extensibility
+    if os.getenv("PAPERLESS_DBENGINE") == "mariadb":
+        engine = "django.db.backends.mysql"
+        options = {"read_default_file": "/etc/mysql/my.cnf"}
+    else:  # Default to PostgresDB
+        engine = "django.db.backends.postgresql_psycopg2"
+        options = {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")}
+
+    DATABASES["default"]["ENGINE"] = engine
+    for key, value in options.items():
+        DATABASES["default"]["OPTIONS"][key] = value
+
 if os.getenv("PAPERLESS_DB_TIMEOUT") is not None:
    _new_opts = {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}
    if "OPTIONS" in DATABASES["default"]: