From 65cfd55027eb14adb26498f4f92736612d0d1e77 Mon Sep 17 00:00:00 2001 From: Brian Kelly Date: Tue, 3 May 2022 20:13:13 -0600 Subject: [PATCH 01/11] rebase mariadb feature onto dev --- Dockerfile | 2 + .../compose/docker-compose.mariadb-tika.yml | 101 ++++++++++++++++++ docker/compose/docker-compose.mariadb.yml | 84 +++++++++++++++ docker/docker-prepare.sh | 28 ++++- docs/administration.rst | 6 +- docs/configuration.rst | 14 ++- docs/extending.rst | 2 +- docs/setup.rst | 7 +- install-paperless-ngx.sh | 23 +++- src/paperless/settings.py | 15 ++- 10 files changed, 268 insertions(+), 14 deletions(-) create mode 100644 docker/compose/docker-compose.mariadb-tika.yml create mode 100644 docker/compose/docker-compose.mariadb.yml diff --git a/Dockerfile b/Dockerfile index 01adb47f5..d0acec101 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,7 @@ ARG RUNTIME_PACKAGES="\ curl \ file \ # fonts for text file thumbnail generation + default-libmysqlclient-dev \ fonts-liberation \ gettext \ ghostscript \ @@ -186,6 +187,7 @@ COPY Pipfile* ./ ARG BUILD_PACKAGES="\ build-essential \ git \ + mariadb-client \ python3-dev" RUN set -eux \ diff --git a/docker/compose/docker-compose.mariadb-tika.yml b/docker/compose/docker-compose.mariadb-tika.yml new file mode 100644 index 000000000..238068184 --- /dev/null +++ b/docker/compose/docker-compose.mariadb-tika.yml @@ -0,0 +1,101 @@ +# docker-compose file for running paperless from the Docker Hub. +# This file contains everything paperless needs to run. +# Paperless supports amd64, arm and arm64 hardware. +# +# All compose files of paperless configure paperless in the following way: +# +# - Paperless is (re)started on system boot, if it was running before shutdown. +# - Docker volumes for storing data are managed by Docker. +# - Folders for importing and exporting files are created in the same directory +# as this file and mounted to the correct folders inside the container. +# - Paperless listens on port 8000. +# +# In addition to that, this docker-compose file adds the following optional +# configurations: +# +# - Instead of SQLite (default), MariaDB is used as the database server. +# - Apache Tika and Gotenberg servers are started with paperless and paperless +# is configured to use these services. These provide support for consuming +# Office documents (Word, Excel, Power Point and their LibreOffice counter- +# parts. +# +# To install and update paperless with this file, do the following: +# +# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env' +# and '.env' into a folder. +# - Run 'docker-compose pull'. +# - Run 'docker-compose run --rm webserver createsuperuser' to create a user. +# - Run 'docker-compose up -d'. +# +# For more extensive installation and update instructions, refer to the +# documentation. + +version: "3.4" +services: + broker: + image: redis:6.0 + restart: unless-stopped + volumes: + - redisdata:/data + + db: + image: mariadb:10 + restart: unless-stopped + volumes: + - dbdata:/var/lib/mariadb/data + environment: + MARIADB_HOST: paperless + MARIADB_DATABASE: paperless + MARIADB_USER: paperless + MARIADB_PASSWORD: paperless + MARIADB_ROOT_PASSWORD: paperless + ports: + - "3306:3306" + + webserver: + image: ghcr.io/paperless-ngx/paperless-ngx:latest + restart: unless-stopped + depends_on: + - db + - broker + - gotenberg + - tika + ports: + - 8000:8000 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000"] + interval: 30s + timeout: 10s + retries: 5 + volumes: + - data:/usr/src/paperless/data + - media:/usr/src/paperless/media + - ./export:/usr/src/paperless/export + - ./consume:/usr/src/paperless/consume + env_file: docker-compose.env + environment: + PAPERLESS_REDIS: redis://broker:6379 + PAPERLESS_DBENGINE: mariadb + PAPERLESS_DBHOST: db + PAPERLESS_DBUSER: root + PAPERLESS_DBPASSWORD: paperless + PAPERLESS_DBPORT: 3306 + PAPERLESS_TIKA_ENABLED: 1 + PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000 + PAPERLESS_TIKA_ENDPOINT: http://tika:9998 + + gotenberg: + image: gotenberg/gotenberg:7 + restart: unless-stopped + environment: + CHROMIUM_DISABLE_ROUTES: 1 + + tika: + image: apache/tika + restart: unless-stopped + +volumes: + data: + media: + dbdata: + redisdata: diff --git a/docker/compose/docker-compose.mariadb.yml b/docker/compose/docker-compose.mariadb.yml new file mode 100644 index 000000000..c8782a20b --- /dev/null +++ b/docker/compose/docker-compose.mariadb.yml @@ -0,0 +1,84 @@ +# docker-compose file for running paperless from the Docker Hub. +# This file contains everything paperless needs to run. +# Paperless supports amd64, arm and arm64 hardware. +# +# All compose files of paperless configure paperless in the following way: +# +# - Paperless is (re)started on system boot, if it was running before shutdown. +# - Docker volumes for storing data are managed by Docker. +# - Folders for importing and exporting files are created in the same directory +# as this file and mounted to the correct folders inside the container. +# - Paperless listens on port 8000. +# +# In addition to that, this docker-compose file adds the following optional +# configurations: +# +# - Instead of SQLite (default), MariaDB is used as the database server. +# +# To install and update paperless with this file, do the following: +# +# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env' +# and '.env' into a folder. +# - Run 'docker-compose pull'. +# - Run 'docker-compose run --rm webserver createsuperuser' to create a user. +# - Run 'docker-compose up -d'. +# +# For more extensive installation and update instructions, refer to the +# documentation. + +version: "3.4" +services: + broker: + image: redis:6.0 + restart: unless-stopped + volumes: + - redisdata:/data + + db: + image: mariadb:10 + restart: unless-stopped + volumes: + - dbdata:/var/lib/mariadb/data + environment: + MARIADB_HOST: paperless + MARIADB_DATABASE: paperless + MARIADB_USER: paperless + MARIADB_PASSWORD: paperless + MARIADB_ROOT_PASSWORD: paperless + ports: + - "3306:3306" + + webserver: +# image: ghcr.io/paperless-ngx/paperless-ngx:latest + image: paperless-bktest:latest + restart: unless-stopped + depends_on: + - db + - broker + ports: + - 8000:8000 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000"] + interval: 30s + timeout: 10s + retries: 5 + volumes: + - data:/usr/src/paperless/data + - media:/usr/src/paperless/media + - ./export:/usr/src/paperless/export + - ./consume:/usr/src/paperless/consume + env_file: docker-compose.env + environment: + PAPERLESS_REDIS: redis://broker:6379 + PAPERLESS_DBENGINE: mariadb + PAPERLESS_DBHOST: db + PAPERLESS_DBUSER: root + PAPERLESS_DBPASSWORD: paperless + PAPERLESS_DBPORT: 3306 + + +volumes: + data: + media: + dbdata: + redisdata: diff --git a/docker/docker-prepare.sh b/docker/docker-prepare.sh index 879e1653a..193811d01 100755 --- a/docker/docker-prepare.sh +++ b/docker/docker-prepare.sh @@ -28,6 +28,30 @@ wait_for_postgres() { done } +wait_for_mariadb() { + echo "Waiting for MariaDB to start..." + + host="${PAPERLESS_DBHOST:=localhost}" + port="${PAPERLESS_DBPORT:=3306}" + + attempt_num=1 + max_attempts=5 + + while ! Defaults to redis://localhost:6379. +PAPERLESS_DBENGINE= + Optional, gives the ability to choose Postgres or MariaDB for database engine. + Available options are `postgresql` and `mariadb`. + Default is `postgresql`. + PAPERLESS_DBHOST= By default, sqlite is used as the database backend. This can be changed here. - Set PAPERLESS_DBHOST and PostgreSQL will be used instead of sqlite. + + Set PAPERLESS_DBHOST and another database will be used instead of sqlite. PAPERLESS_DBPORT= Adjust port if necessary. @@ -39,17 +45,17 @@ PAPERLESS_DBPORT= Default is 5432. PAPERLESS_DBNAME= - Database name in PostgreSQL. + Database name in PostgreSQL or MariaDB. Defaults to "paperless". PAPERLESS_DBUSER= - Database user in PostgreSQL. + Database user in PostgreSQL or MariaDB. Defaults to "paperless". PAPERLESS_DBPASS= - Database password for PostgreSQL. + Database password for PostgreSQL or MariaDB. Defaults to "paperless". diff --git a/docs/extending.rst b/docs/extending.rst index 199cd78ce..bfc289689 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -79,7 +79,7 @@ To do the setup you need to perform the steps from the following chapters in a c 6. You can now either ... * install redis or - * use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a postgresql server) or + * use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a database server) or * spin up a bare redis container .. code:: shell-session diff --git a/docs/setup.rst b/docs/setup.rst index 3aea55467..9312e70d4 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -73,7 +73,7 @@ Paperless consists of the following components: for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different process (maybe even on different machines!), and therefore, this is necessary. -* Optional: A database server. Paperless supports both PostgreSQL and SQLite for storing its data. +* Optional: A database server. Paperless supports PostgreSQL, MariaDB and SQLite for storing its data. Installation @@ -316,11 +316,13 @@ writing. Windows is not and will never be supported. * ``python3-pip`` * ``python3-dev`` + * ``default-libmysqlclient-dev`` for MariaDB * ``fonts-liberation`` for generating thumbnails for plain text files * ``imagemagick`` >= 6 for PDF conversion * ``gnupg`` for handling encrypted documents * ``libpq-dev`` for PostgreSQL * ``libmagic-dev`` for mime type detection + * ``mariadb-client`` for MariaDB compile time * ``mime-support`` for mime type detection * ``libzbar0`` for barcode detection * ``poppler-utils`` for barcode detection @@ -361,7 +363,7 @@ writing. Windows is not and will never be supported. 2. Install ``redis`` >= 5.0 and configure it to start automatically. 3. Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish - to use PostgreSQL, SQLite is available as well. + to use PostgreSQL, MariaDB and SQLite are available as well. .. note:: @@ -377,6 +379,7 @@ writing. Windows is not and will never be supported. settings to your needs. Required settings for getting paperless running are: * ``PAPERLESS_REDIS`` should point to your redis server, such as redis://localhost:6379. + * ``PAPERLESS_DBENGINE`` optional, and should be one of `postgres, mariadb, or sqlite` * ``PAPERLESS_DBHOST`` should be the hostname on which your PostgreSQL server is running. Do not configure this to use SQLite instead. Also configure port, database name, user and password as necessary. * ``PAPERLESS_CONSUMPTION_DIR`` should point to a folder which paperless should watch for documents. You might diff --git a/install-paperless-ngx.sh b/install-paperless-ngx.sh index fec14ea83..d0bc0d59e 100755 --- a/install-paperless-ngx.sh +++ b/install-paperless-ngx.sh @@ -118,12 +118,12 @@ ask "Current time zone" "$default_time_zone" TIME_ZONE=$ask_result echo "" -echo "Database backend: PostgreSQL and SQLite are available. Use PostgreSQL" +echo "Database backend: PostgreSQL, MariaDB, and SQLite are available. Use PostgreSQL" echo "if unsure. If you're running on a low-power device such as Raspberry" echo "Pi, use SQLite to save resources." echo "" -ask "Database backend" "postgres" "postgres sqlite" +ask "Database backend" "postgres" "postgres sqlite" "mariadb" DATABASE_BACKEND=$ask_result echo "" @@ -227,6 +227,18 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then POSTGRES_FOLDER=$ask_result fi +if [[ "$DATABASE_BACKEND" == "mariadb" ]] ; then + echo "" + echo "The database folder, where mariadb stores its data." + echo "" + echo "CAUTION: If specified, you must specify an absolute path starting with /" + echo "or a relative path starting with ./ here." + echo "" + + ask_docker_folder "Database folder" "" + MARIADB_FOLDER=$ask_result +fi + echo "" echo "3. Login credentials" echo "====================" @@ -285,6 +297,9 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then echo "Database (postgres) folder: $POSTGRES_FOLDER" fi fi +if [[ "$DATABASE_BACKEND" == "mariadb" ]] ; then + echo "Database (mariadb) folder: $MARIADB_FOLDER" +fi echo "" echo "URL: $URL" echo "Port: $PORT" @@ -361,6 +376,10 @@ if [[ -n $POSTGRES_FOLDER ]] ; then sed -i "/^\s*pgdata:/d" docker-compose.yml fi +if [[ -n $MARIADB_FOLDER ]] ; then + sed -i "s#- dbdata:/var/lib/mariadb/data#- $MARIADB_FOLDER:/var/lib/mariadb/data#g" docker-compose.yml +fi + # remove trailing blank lines from end of file sed -i -e :a -e '/^\n*$/{$d;N;};/\n$/ba' docker-compose.yml # if last line in file contains "volumes:", remove that line since no more named volumes are left diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 92042ac31..ff9d350ce 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -328,16 +328,27 @@ if os.getenv("PAPERLESS_DBHOST"): DATABASES["sqlite"] = DATABASES["default"].copy() DATABASES["default"] = { - "ENGINE": "django.db.backends.postgresql_psycopg2", "HOST": os.getenv("PAPERLESS_DBHOST"), "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"), "USER": os.getenv("PAPERLESS_DBUSER", "paperless"), "PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"), - "OPTIONS": {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")}, + "OPTIONS": {}, } if os.getenv("PAPERLESS_DBPORT"): DATABASES["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT") + # Leave room for future extensibility + if os.getenv("PAPERLESS_DBENGINE") == "mariadb": + engine = "django.db.backends.mysql" + options = {"read_default_file": "/etc/mysql/my.cnf"} + else: # Default to PostgresDB + engine = "django.db.backends.postgresql_psycopg2" + options = {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")} + + DATABASES["default"]["ENGINE"] = engine + for key, value in options.items(): + DATABASES["default"]["OPTIONS"][key] = value + if os.getenv("PAPERLESS_DB_TIMEOUT") is not None: _new_opts = {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))} if "OPTIONS" in DATABASES["default"]: From 6ae9a8f2bea39e8441357a7e5558549806547fb4 Mon Sep 17 00:00:00 2001 From: Brian Kelly Date: Tue, 3 May 2022 20:13:41 -0600 Subject: [PATCH 02/11] remove debug image reference --- docker/compose/docker-compose.mariadb.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/compose/docker-compose.mariadb.yml b/docker/compose/docker-compose.mariadb.yml index c8782a20b..5242e18bf 100644 --- a/docker/compose/docker-compose.mariadb.yml +++ b/docker/compose/docker-compose.mariadb.yml @@ -49,8 +49,7 @@ services: - "3306:3306" webserver: -# image: ghcr.io/paperless-ngx/paperless-ngx:latest - image: paperless-bktest:latest + image: ghcr.io/paperless-ngx/paperless-ngx:latest restart: unless-stopped depends_on: - db From f528b01de4a81196601d9013f265a411b6ca2962 Mon Sep 17 00:00:00 2001 From: Brian Kelly Date: Thu, 12 May 2022 21:01:47 -0600 Subject: [PATCH 03/11] set paperless db user --- docker/compose/docker-compose.mariadb-tika.yml | 2 +- docker/compose/docker-compose.mariadb.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/compose/docker-compose.mariadb-tika.yml b/docker/compose/docker-compose.mariadb-tika.yml index 238068184..8a4a72040 100644 --- a/docker/compose/docker-compose.mariadb-tika.yml +++ b/docker/compose/docker-compose.mariadb-tika.yml @@ -77,7 +77,7 @@ services: PAPERLESS_REDIS: redis://broker:6379 PAPERLESS_DBENGINE: mariadb PAPERLESS_DBHOST: db - PAPERLESS_DBUSER: root + PAPERLESS_DBUSER: paperless PAPERLESS_DBPASSWORD: paperless PAPERLESS_DBPORT: 3306 PAPERLESS_TIKA_ENABLED: 1 diff --git a/docker/compose/docker-compose.mariadb.yml b/docker/compose/docker-compose.mariadb.yml index 5242e18bf..dd95bf7b8 100644 --- a/docker/compose/docker-compose.mariadb.yml +++ b/docker/compose/docker-compose.mariadb.yml @@ -71,7 +71,7 @@ services: PAPERLESS_REDIS: redis://broker:6379 PAPERLESS_DBENGINE: mariadb PAPERLESS_DBHOST: db - PAPERLESS_DBUSER: root + PAPERLESS_DBUSER: paperless PAPERLESS_DBPASSWORD: paperless PAPERLESS_DBPORT: 3306 From ebdf9b55df21bd3619887563fbcc7db28d9f38a6 Mon Sep 17 00:00:00 2001 From: Brian Kelly Date: Tue, 17 May 2022 12:38:02 -0600 Subject: [PATCH 04/11] correct the mariadb volume data path --- docker/compose/docker-compose.mariadb-tika.yml | 2 +- docker/compose/docker-compose.mariadb.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/compose/docker-compose.mariadb-tika.yml b/docker/compose/docker-compose.mariadb-tika.yml index 8a4a72040..cf7bae6c4 100644 --- a/docker/compose/docker-compose.mariadb-tika.yml +++ b/docker/compose/docker-compose.mariadb-tika.yml @@ -42,7 +42,7 @@ services: image: mariadb:10 restart: unless-stopped volumes: - - dbdata:/var/lib/mariadb/data + - dbdata:/var/lib/mysql environment: MARIADB_HOST: paperless MARIADB_DATABASE: paperless diff --git a/docker/compose/docker-compose.mariadb.yml b/docker/compose/docker-compose.mariadb.yml index dd95bf7b8..ac895d60b 100644 --- a/docker/compose/docker-compose.mariadb.yml +++ b/docker/compose/docker-compose.mariadb.yml @@ -38,7 +38,7 @@ services: image: mariadb:10 restart: unless-stopped volumes: - - dbdata:/var/lib/mariadb/data + - dbdata:/var/lib/mysql environment: MARIADB_HOST: paperless MARIADB_DATABASE: paperless From 3b84e34c8e8e7ad102c6ac3e5f2a2b38559e8594 Mon Sep 17 00:00:00 2001 From: Brian Kelly Date: Tue, 31 May 2022 22:18:43 -0600 Subject: [PATCH 05/11] re-order apt dependencies --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d0acec101..60f3374e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,7 +56,6 @@ ARG RUNTIME_PACKAGES="\ curl \ file \ # fonts for text file thumbnail generation - default-libmysqlclient-dev \ fonts-liberation \ gettext \ ghostscript \ @@ -78,10 +77,12 @@ ARG RUNTIME_PACKAGES="\ libraqm0 \ libgnutls30 \ libjpeg62-turbo \ + optipng \ python3 \ python3-pip \ python3-setuptools \ postgresql-client \ + mariadb-client \ # For Numpy libatlas3-base \ # OCRmyPDF dependencies @@ -187,7 +188,7 @@ COPY Pipfile* ./ ARG BUILD_PACKAGES="\ build-essential \ git \ - mariadb-client \ + default-libmysqlclient-dev \ python3-dev" RUN set -eux \ From 60980cb26ae954f082441e31248f04657d327375 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Tue, 30 Aug 2022 14:56:14 -0700 Subject: [PATCH 06/11] Adds the mysql Python dependency --- Pipfile | 1 + Pipfile.lock | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index 87cf55f3f..48b855f1a 100644 --- a/Pipfile +++ b/Pipfile @@ -52,6 +52,7 @@ concurrent-log-handler = "*" zipp = {version = "*", markers = "python_version < '3.9'"} pyzbar = "*" pdf2image = "*" +mysqlclient = "*" [dev-packages] coveralls = "*" diff --git a/Pipfile.lock b/Pipfile.lock index bf59cc8e2..b439beadd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -697,6 +697,19 @@ ], "version": "==1.0.4" }, + "mysqlclient": { + "hashes": [ + "sha256:0d1cd3a5a4d28c222fa199002810e8146cffd821410b67851af4cc80aeccd97c", + "sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782", + "sha256:996924f3483fd36a34a5812210c69e71dea5a3d5978d01199b78b7f6d485c855", + "sha256:b355c8b5a7d58f2e909acdbb050858390ee1b0e13672ae759e5e784110022994", + "sha256:c1ed71bd6244993b526113cca3df66428609f90e4652f37eb51c33496d478b37", + "sha256:c812b67e90082a840efb82a8978369e6e69fc62ce1bda4ca8f3084a9d862308b", + "sha256:dea88c8d3f5a5d9293dfe7f087c16dd350ceb175f2f6631c9cf4caf3e19b7a96" + ], + "index": "pypi", + "version": "==2.1.1" + }, "numpy": { "hashes": [ "sha256:17e5226674f6ea79e14e3b91bfbc153fdf3ac13f5cc54ee7bc8fdbe820a32da0", @@ -1764,9 +1777,7 @@ "version": "==0.4.5" }, "coverage": { - "extras": [ - "toml" - ], + "extras": [], "hashes": [ "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2", "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820", From ed2e884de8bd7d1b4657d45ee4b6568f0c8b62df Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Tue, 30 Aug 2022 14:58:20 -0700 Subject: [PATCH 07/11] Fully qualifies the image and updates to Redis 7 --- docker/compose/docker-compose.mariadb-tika.yml | 8 ++++---- docker/compose/docker-compose.mariadb.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/compose/docker-compose.mariadb-tika.yml b/docker/compose/docker-compose.mariadb-tika.yml index cf7bae6c4..42ad74051 100644 --- a/docker/compose/docker-compose.mariadb-tika.yml +++ b/docker/compose/docker-compose.mariadb-tika.yml @@ -33,13 +33,13 @@ version: "3.4" services: broker: - image: redis:6.0 + image: docker.io/library/redis:7 restart: unless-stopped volumes: - redisdata:/data db: - image: mariadb:10 + image: docker.io/library/mariadb:10 restart: unless-stopped volumes: - dbdata:/var/lib/mysql @@ -85,13 +85,13 @@ services: PAPERLESS_TIKA_ENDPOINT: http://tika:9998 gotenberg: - image: gotenberg/gotenberg:7 + image: docker.io/gotenberg/gotenberg:7.4 restart: unless-stopped environment: CHROMIUM_DISABLE_ROUTES: 1 tika: - image: apache/tika + image: ghcr.io/paperless-ngx/tika:latest restart: unless-stopped volumes: diff --git a/docker/compose/docker-compose.mariadb.yml b/docker/compose/docker-compose.mariadb.yml index ac895d60b..8cb452464 100644 --- a/docker/compose/docker-compose.mariadb.yml +++ b/docker/compose/docker-compose.mariadb.yml @@ -29,13 +29,13 @@ version: "3.4" services: broker: - image: redis:6.0 + image: docker.io/library/redis:7 restart: unless-stopped volumes: - redisdata:/data db: - image: mariadb:10 + image: docker.io/library/mariadb:10 restart: unless-stopped volumes: - dbdata:/var/lib/mysql From 537e7c63f4461d8bdb7c1fa9d170b0914ff91b11 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Tue, 30 Aug 2022 15:17:08 -0700 Subject: [PATCH 08/11] Combines the two database folders to a single variable during installation --- install-paperless-ngx.sh | 51 ++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/install-paperless-ngx.sh b/install-paperless-ngx.sh index d0bc0d59e..d5266efca 100755 --- a/install-paperless-ngx.sh +++ b/install-paperless-ngx.sh @@ -123,7 +123,7 @@ echo "if unsure. If you're running on a low-power device such as Raspberry" echo "Pi, use SQLite to save resources." echo "" -ask "Database backend" "postgres" "postgres sqlite" "mariadb" +ask "Database backend" "postgres" "postgres sqlite mariadb" DATABASE_BACKEND=$ask_result echo "" @@ -214,9 +214,9 @@ echo "" ask_docker_folder "Data folder" "" DATA_FOLDER=$ask_result -if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then +if [[ "$DATABASE_BACKEND" == "postgres" || "$DATABASE_BACKEND" == "mariadb" ]] ; then echo "" - echo "The database folder, where postgres stores its data." + echo "The database folder, where your database stores its data." echo "Leave empty to have this managed by docker." echo "" echo "CAUTION: If specified, you must specify an absolute path starting with /" @@ -224,19 +224,7 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then echo "" ask_docker_folder "Database folder" "" - POSTGRES_FOLDER=$ask_result -fi - -if [[ "$DATABASE_BACKEND" == "mariadb" ]] ; then - echo "" - echo "The database folder, where mariadb stores its data." - echo "" - echo "CAUTION: If specified, you must specify an absolute path starting with /" - echo "or a relative path starting with ./ here." - echo "" - - ask_docker_folder "Database folder" "" - MARIADB_FOLDER=$ask_result + DATABASE_FOLDER=$ask_result fi echo "" @@ -290,16 +278,14 @@ if [[ -z $DATA_FOLDER ]] ; then else echo "Data folder: $DATA_FOLDER" fi -if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then - if [[ -z $POSTGRES_FOLDER ]] ; then - echo "Database (postgres) folder: Managed by docker" +if [[ "$DATABASE_BACKEND" == "postgres" || "$DATABASE_BACKEND" == "mariadb" ]] ; then + if [[ -z $DATABASE_FOLDER ]] ; then + echo "Database folder: Managed by docker" else - echo "Database (postgres) folder: $POSTGRES_FOLDER" + echo "Database folder: $DATABASE_FOLDER" fi fi -if [[ "$DATABASE_BACKEND" == "mariadb" ]] ; then - echo "Database (mariadb) folder: $MARIADB_FOLDER" -fi + echo "" echo "URL: $URL" echo "Port: $PORT" @@ -371,13 +357,16 @@ if [[ -n $DATA_FOLDER ]] ; then sed -i "/^\s*data:/d" docker-compose.yml fi -if [[ -n $POSTGRES_FOLDER ]] ; then - sed -i "s#- pgdata:/var/lib/postgresql/data#- $POSTGRES_FOLDER:/var/lib/postgresql/data#g" docker-compose.yml - sed -i "/^\s*pgdata:/d" docker-compose.yml -fi - -if [[ -n $MARIADB_FOLDER ]] ; then - sed -i "s#- dbdata:/var/lib/mariadb/data#- $MARIADB_FOLDER:/var/lib/mariadb/data#g" docker-compose.yml +# If the database folder was provided (not blank), replace the pgdata/dbdata volume with a bind mount +# of the provided folder +if [[ -n $DATABASE_FOLDER ]] ; then + if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then + sed -i "s#- pgdata:/var/lib/postgresql/data#- $DATABASE_FOLDER:/var/lib/postgresql/data#g" docker-compose.yml + sed -i "/^\s*pgdata:/d" docker-compose.yml + elif [[ "$DATABASE_BACKEND" == "mariadb" ]]; then + sed -i "s#- dbdata:/var/lib/mysql#- $DATABASE_FOLDER:/var/lib/mysql#g" docker-compose.yml + sed -i "/^\s*dbdata:/d" docker-compose.yml + fi fi # remove trailing blank lines from end of file @@ -394,4 +383,4 @@ ${DOCKER_COMPOSE_CMD} pull ${DOCKER_COMPOSE_CMD} run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL" -${DOCKER_COMPOSE_CMD} up -d +${DOCKER_COMPOSE_CMD} up --detach From 140e239bdb61f49cfa834408862f317a82a12a4c Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Tue, 30 Aug 2022 15:34:56 -0700 Subject: [PATCH 09/11] Fixes the shellcheck issues --- docker/docker-prepare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-prepare.sh b/docker/docker-prepare.sh index 193811d01..c4e45c032 100755 --- a/docker/docker-prepare.sh +++ b/docker/docker-prepare.sh @@ -37,7 +37,7 @@ wait_for_mariadb() { attempt_num=1 max_attempts=5 - while ! /dev/tcp/$host/$port; do if [ $attempt_num -eq $max_attempts ]; then echo "Unable to connect to database." @@ -47,7 +47,7 @@ wait_for_mariadb() { fi - attempt_num=$(expr "$attempt_num" + 1) + attempt_num=$(("$attempt_num" + 1)) sleep 5 done } From 18a4ba77780a0628e4be61b725969f86f1438d0d Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Tue, 30 Aug 2022 15:38:57 -0700 Subject: [PATCH 10/11] Removes optipng, added in a bad merge --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 60f3374e7..bb9a148ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,7 +77,6 @@ ARG RUNTIME_PACKAGES="\ libraqm0 \ libgnutls30 \ libjpeg62-turbo \ - optipng \ python3 \ python3-pip \ python3-setuptools \ From 2c3c26edf18a458eff6c9549d647dc7224bb8fe6 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Sun, 11 Sep 2022 08:20:58 -0700 Subject: [PATCH 11/11] Ensures the Pipfile.lock hash is up to date --- Pipfile.lock | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index b439beadd..651ae4825 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8d8183effbecd0c91106f23f66517f720754f7d129eec22c86969da43f69ca82" + "sha256": "af0a0a5b996c11ad95266e98200640fd77648ec0f337ac3eb4e4b5ca1c714a0e" }, "pipfile-spec": 6, "requires": {}, @@ -39,7 +39,7 @@ "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1", "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.2.3" }, "asgiref": { @@ -55,7 +55,7 @@ "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15", "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.0.2" }, "attrs": { @@ -115,7 +115,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.6.15" }, "cffi": { @@ -208,7 +208,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "click": { @@ -271,7 +271,7 @@ "sha256:d4ef6cc305394ed669d4d9eebf10d3a101059bdcf2669c366ec1d14e4fb227bd", "sha256:d9e69ae01f99abe6ad646947bba8941e896cb3aa805be2597a0400e0764b5818" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==38.0.1" }, "daphne": { @@ -279,7 +279,7 @@ "sha256:76ffae916ba3aa66b46996c14fa713e46004788167a4873d647544e750e0e99f", "sha256:a9af943c79717bc52fe64a3c236ae5d3adccc8b5be19c881b442d2c3db233393" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==3.0.2" }, "dateparser": { @@ -390,7 +390,7 @@ "sha256:70813c1135087a248a4d38cc0e1a0181ffab2188141a93eaf567940c3957ff06", "sha256:8ddd78563b633ca55346c8cd41ec0af27d3c79931828beffb46ce70a379e7442" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==0.13.0" }, "hiredis": { @@ -437,7 +437,7 @@ "sha256:f52010e0a44e3d8530437e7da38d11fb822acfb0d5b12e9cd5ba655509937ca0", "sha256:f8196f739092a78e4f6b1b2172679ed3343c39c61a3e9d722ce6fcf1dac2824a" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.0.0" }, "httptools": { @@ -553,7 +553,7 @@ "sha256:4158fcecd13733f8be669be0683b96ebdbbd38d23559f54dca7205aea1bf1e35", "sha256:f21f109b3c7ff9d95f8387f752d0d9c34a02aa2f7060c2135f465da0e5160ff6" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.1.0" }, "langdetect": { @@ -757,7 +757,7 @@ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==21.3" }, "pathvalidate": { @@ -894,7 +894,7 @@ "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.0.0" }, "portalocker": { @@ -1165,7 +1165,7 @@ "sha256:fbc88d3ba402b5d041d204ec2449c4078898f89c4a6e6f0ed1c1a510ef1e221d", "sha256:fbd3fe37353c62fd0eb19fb76f78aa693716262bcd5f9c14bb9e5aca4b3f0dc4" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.3.2" }, "reportlab": { @@ -1315,7 +1315,7 @@ "sha256:8b99adda265feb6773280df41eece7b2e6561b772d21ffd52e372f999024907b", "sha256:a335baacfaa4400ae1f0d8e3a58d6674d2f8828e3716bb2802c44955ad391380" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==3.1.0" }, "tika": { @@ -1349,7 +1349,7 @@ "sha256:2e4582b70f04b2345908254684a984206c0d9b50e3074a24a4c55aba21d24d01", "sha256:41223af4a9d5726e645a8ee82480f413e5e300dd257db94bc38ae12ea48fb2e5" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==22.2.1" }, "typing-extensions": { @@ -1365,7 +1365,7 @@ "sha256:21f4f0d7241572efa7f7a4fdabb052e61b55dc48274e6842697ccdf5253e5451", "sha256:c3119520447d68ef3eb8187a55a4f44fa455f30eb1b4238fa5691ba094f2b05b" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.2" }, "tzlocal": { @@ -1373,7 +1373,7 @@ "sha256:89885494684c929d9191c57aa27502afc87a579be5cdd3225c77c463ea043745", "sha256:ee5842fa3a795f023514ac2d801c4a81d1743bbe642e3940143326b3a00addd7" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.2" }, "urllib3": { @@ -1741,7 +1741,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.6.15" }, "cfgv": { @@ -1757,7 +1757,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "click": { @@ -1777,7 +1777,6 @@ "version": "==0.4.5" }, "coverage": { - "extras": [], "hashes": [ "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2", "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820", @@ -2037,7 +2036,7 @@ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==21.3" }, "pathspec": { @@ -2061,7 +2060,7 @@ "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.0.0" }, "pre-commit": { @@ -2351,7 +2350,7 @@ "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" ], - "markers": "python_version >= '3.7'", + "markers": "python_version > '2.7'", "version": "==6.2" }, "tox": {