Merge branch 'dev' into beta

docker/compose/docker-compose.mariadb-tika.yml

@@ -85,10 +85,11 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.6
     restart: unless-stopped
-    environment:
-      CHROMIUM_DISABLE_ROUTES: 1
+    command:
+      - "gotenberg"
+      - "--chromium-disable-routes=true"
 
   tika:
     image: ghcr.io/paperless-ngx/tika:latest

docker/compose/docker-compose.postgres-tika.yml

@@ -77,7 +77,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.6
     restart: unless-stopped
     command:
       - "gotenberg"

docker/compose/docker-compose.sqlite-tika.yml

@@ -65,7 +65,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.6
     restart: unless-stopped
     command:
       - "gotenberg"

docker/docker-entrypoint.sh

@@ -9,8 +9,8 @@ set -e
 # fill in the value of "$XYZ_DB_PASSWORD" from a file, especially for Docker's
 # secrets feature
 file_env() {
-	local var="$1"
-	local fileVar="${var}_FILE"
+	local -r var="$1"
+	local -r fileVar="${var}_FILE"
 
 	# Basic validation
 	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
@@ -35,14 +35,14 @@ file_env() {
 
 # Source: https://github.com/sameersbn/docker-gitlab/
 map_uidgid() {
-	USERMAP_ORIG_UID=$(id -u paperless)
-	USERMAP_ORIG_GID=$(id -g paperless)
-	USERMAP_NEW_UID=${USERMAP_UID:-$USERMAP_ORIG_UID}
-	USERMAP_NEW_GID=${USERMAP_GID:-${USERMAP_ORIG_GID:-$USERMAP_NEW_UID}}
-	if [[ ${USERMAP_NEW_UID} != "${USERMAP_ORIG_UID}" || ${USERMAP_NEW_GID} != "${USERMAP_ORIG_GID}" ]]; then
-		echo "Mapping UID and GID for paperless:paperless to $USERMAP_NEW_UID:$USERMAP_NEW_GID"
-		usermod -o -u "${USERMAP_NEW_UID}" paperless
-		groupmod -o -g "${USERMAP_NEW_GID}" paperless
+	local -r usermap_original_uid=$(id -u paperless)
+	local -r usermap_original_gid=$(id -g paperless)
+	local -r usermap_new_uid=${USERMAP_UID:-$usermap_original_uid}
+	local -r usermap_new_gid=${USERMAP_GID:-${usermap_original_gid:-$usermap_new_uid}}
+	if [[ ${usermap_new_uid} != "${usermap_original_uid}" || ${usermap_new_gid} != "${usermap_original_gid}" ]]; then
+		echo "Mapping UID and GID for paperless:paperless to $usermap_new_uid:$usermap_new_gid"
+		usermod -o -u "${usermap_new_uid}" paperless
+		groupmod -o -g "${usermap_new_gid}" paperless
 	fi
 }
 
@@ -53,6 +53,30 @@ map_folders() {
 	export CONSUME_DIR="${PAPERLESS_CONSUMPTION_DIR:-/usr/src/paperless/consume}"
 }
 
+nltk_data () {
+	# Store the NLTK data outside the Docker container
+	local -r nltk_data_dir="${DATA_DIR}/nltk"
+	local -r truthy_things=("yes y 1 t true")
+
+	# If not set, or it looks truthy
+	if [[ -z "${PAPERLESS_ENABLE_NLTK}" ]] || [[ "${truthy_things[*]}" =~ ${PAPERLESS_ENABLE_NLTK,} ]]; then
+
+		# Download or update the snowball stemmer data
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" snowball_data
+
+		# Download or update the stopwords corpus
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" stopwords
+
+		# Download or update the punkt tokenizer data
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" punkt
+
+	else
+		echo "Skipping NLTK data download"
+
+	fi
+
+}
+
 initialize() {
 
 	# Setup environment from secrets before anything else
@@ -76,7 +100,7 @@ initialize() {
 	# Check for overrides of certain folders
 	map_folders
 
-	local export_dir="/usr/src/paperless/export"
+	local -r export_dir="/usr/src/paperless/export"
 
 	for dir in \
 		"${export_dir}" \
@@ -89,10 +113,12 @@ initialize() {
 		fi
 	done
 
-	local tmp_dir="/tmp/paperless"
+	local -r tmp_dir="/tmp/paperless"
 	echo "Creating directory ${tmp_dir}"
 	mkdir -p "${tmp_dir}"
 
+	nltk_data
+
 	set +e
 	echo "Adjusting permissions of paperless files. This may take a while."
 	chown -R paperless:paperless ${tmp_dir}
@@ -111,7 +137,7 @@ initialize() {
 install_languages() {
 	echo "Installing languages..."
 
-	local langs="$1"
+	local -r langs="$1"
 	read -ra langs <<<"$langs"
 
 	# Check that it is not empty

docker/docker-prepare.sh

@@ -4,12 +4,12 @@ set -e
 
 wait_for_postgres() {
 	local attempt_num=1
-	local max_attempts=5
+	local -r max_attempts=5
 
 	echo "Waiting for PostgreSQL to start..."
 
-	local host="${PAPERLESS_DBHOST:-localhost}"
-	local port="${PAPERLESS_DBPORT:-5432}"
+	local -r host="${PAPERLESS_DBHOST:-localhost}"
+	local -r port="${PAPERLESS_DBPORT:-5432}"
 
 	# Disable warning, host and port can't have spaces
 	# shellcheck disable=SC2086
@@ -31,11 +31,11 @@ wait_for_postgres() {
 wait_for_mariadb() {
 	echo "Waiting for MariaDB to start..."
 
-	host="${PAPERLESS_DBHOST:=localhost}"
-	port="${PAPERLESS_DBPORT:=3306}"
+	local -r host="${PAPERLESS_DBHOST:=localhost}"
+	local -r port="${PAPERLESS_DBPORT:=3306}"
 
-	attempt_num=1
-	max_attempts=5
+	local attempt_num=1
+	local -r max_attempts=5
 
 	while ! true > /dev/tcp/$host/$port; do
 
@@ -73,8 +73,8 @@ migrations() {
 
 search_index() {
 
-	local index_version=1
-	local index_version_file=${DATA_DIR}/.index_version
+	local -r index_version=1
+	local -r index_version_file=${DATA_DIR}/.index_version
 
 	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then
 		echo "Search index out of date. Updating..."
@@ -89,6 +89,46 @@ superuser() {
 	fi
 }
 
+custom_container_init() {
+	# Mostly borrowed from the LinuxServer.io base image
+	# https://github.com/linuxserver/docker-baseimage-ubuntu/tree/bionic/root/etc/cont-init.d
+	local -r custom_script_dir="/custom-cont-init.d"
+	# Tamper checking.
+	# Don't run files which are owned by anyone except root
+	# Don't run files which are writeable by others
+	if [ -d "${custom_script_dir}" ]; then
+		if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 ! -user root)" ]; then
+			echo "**** Potential tampering with custom scripts detected ****"
+			echo "**** The folder '${custom_script_dir}' must be owned by root ****"
+			return 0
+		fi
+		if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 -perm -o+w)" ]; then
+			echo "**** The folder '${custom_script_dir}' or some of contents have write permissions for others, which is a security risk. ****"
+			echo "**** Please review the permissions and their contents to make sure they are owned by root, and can only be modified by root. ****"
+			return 0
+		fi
+
+		# Make sure custom init directory has files in it
+		if [ -n "$(/bin/ls -A "${custom_script_dir}" 2>/dev/null)" ]; then
+			echo "[custom-init] files found in ${custom_script_dir} executing"
+			# Loop over files in the directory
+			for SCRIPT in "${custom_script_dir}"/*; do
+				NAME="$(basename "${SCRIPT}")"
+				if [ -f "${SCRIPT}" ]; then
+					echo "[custom-init] ${NAME}: executing..."
+					/bin/bash "${SCRIPT}"
+					echo "[custom-init] ${NAME}: exited $?"
+				elif [ ! -f "${SCRIPT}" ]; then
+					echo "[custom-init] ${NAME}: is not a file"
+				fi
+			done
+		else
+			echo "[custom-init] no custom files found exiting..."
+		fi
+
+	fi
+}
+
 do_work() {
 	if [[ "${PAPERLESS_DBENGINE}" == "mariadb" ]]; then
 		wait_for_mariadb
@@ -104,6 +144,9 @@ do_work() {
 
 	superuser
 
+	# Leave this last thing
+	custom_container_init
+
 }
 
 do_work

docker/flower-conditional.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "Checking if we should start flower..."
+
+if [[ -n  "${PAPERLESS_ENABLE_FLOWER}" ]]; then
+	celery --app paperless flower
+fi

docker/supervisord.conf

@@ -10,7 +10,7 @@ user=root
 [program:gunicorn]
 command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.asgi:application
 user=paperless
-
+priority = 1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
@@ -20,17 +20,40 @@ stderr_logfile_maxbytes=0
 command=python3 manage.py document_consumer
 user=paperless
 stopsignal=INT
-
+priority = 20
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
-[program:scheduler]
-command=python3 manage.py qcluster
+[program:celery]
+
+command = celery --app paperless worker --loglevel INFO
 user=paperless
 stopasgroup = true
-
+stopwaitsecs = 60
+priority = 5
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:celery-beat]
+
+command = celery --app paperless beat --loglevel INFO
+user=paperless
+stopasgroup = true
+priority = 10
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:celery-flower]
+command = /usr/local/bin/flower-conditional.sh
+user = paperless
+startsecs = 0
+priority = 40
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr