Merge remote-tracking branch 'paperless/dev' into feature-consume-eml

docker/compose/docker-compose.portainer.yml

@@ -31,13 +31,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data

docker/compose/docker-compose.postgres-tika.yml

@@ -33,13 +33,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
@@ -77,7 +77,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.4
     restart: unless-stopped
 
   tika:

docker/compose/docker-compose.postgres.yml

@@ -29,13 +29,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data

docker/compose/docker-compose.sqlite-tika.yml

@@ -33,7 +33,7 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
@@ -65,7 +65,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.4
     restart: unless-stopped
 
   tika:

docker/compose/docker-compose.sqlite.yml

@@ -26,7 +26,7 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data

docker/docker-entrypoint.sh

@@ -2,6 +2,37 @@
 
 set -e
 
+# Adapted from:
+# https://github.com/docker-library/postgres/blob/master/docker-entrypoint.sh
+# usage: file_env VAR
+#    ie: file_env 'XYZ_DB_PASSWORD' will allow for "$XYZ_DB_PASSWORD_FILE" to
+# fill in the value of "$XYZ_DB_PASSWORD" from a file, especially for Docker's
+# secrets feature
+file_env() {
+	local var="$1"
+	local fileVar="${var}_FILE"
+
+	# Basic validation
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+		exit 1
+	fi
+
+	# Only export var if the _FILE exists
+	if [ "${!fileVar:-}" ]; then
+		# And the file exists
+		if [[ -f ${!fileVar} ]]; then
+			echo "Setting ${var} from file"
+			val="$(< "${!fileVar}")"
+			export "$var"="$val"
+		else
+			echo "File ${!fileVar} doesn't exist"
+			exit 1
+		fi
+	fi
+
+}
+
 # Source: https://github.com/sameersbn/docker-gitlab/
 map_uidgid() {
 	USERMAP_ORIG_UID=$(id -u paperless)
@@ -15,23 +46,53 @@ map_uidgid() {
 	fi
 }
 
+map_folders() {
+	# Export these so they can be used in docker-prepare.sh
+	export DATA_DIR="${PAPERLESS_DATA_DIR:-/usr/src/paperless/data}"
+	export MEDIA_ROOT_DIR="${PAPERLESS_MEDIA_ROOT:-/usr/src/paperless/media}"
+}
+
 initialize() {
+
+	# Setup environment from secrets before anything else
+	for env_var in \
+		PAPERLESS_DBUSER \
+		PAPERLESS_DBPASS \
+		PAPERLESS_SECRET_KEY \
+		PAPERLESS_AUTO_LOGIN_USERNAME \
+		PAPERLESS_ADMIN_USER \
+		PAPERLESS_ADMIN_MAIL \
+		PAPERLESS_ADMIN_PASSWORD; do
+		# Check for a version of this var with _FILE appended
+		# and convert the contents to the env var value
+		file_env ${env_var}
+	done
+
+	# Change the user and group IDs if needed
 	map_uidgid
 
-	for dir in export data data/index media media/documents media/documents/originals media/documents/thumbnails; do
-		if [[ ! -d "../$dir" ]]; then
-			echo "Creating directory ../$dir"
-			mkdir ../$dir
+	# Check for overrides of certain folders
+	map_folders
+
+	local export_dir="/usr/src/paperless/export"
+
+	for dir in "${export_dir}" "${DATA_DIR}" "${DATA_DIR}/index" "${MEDIA_ROOT_DIR}" "${MEDIA_ROOT_DIR}/documents" "${MEDIA_ROOT_DIR}/documents/originals" "${MEDIA_ROOT_DIR}/documents/thumbnails"; do
+		if [[ ! -d "${dir}" ]]; then
+			echo "Creating directory ${dir}"
+			mkdir "${dir}"
 		fi
 	done
 
-	echo "Creating directory /tmp/paperless"
-	mkdir -p /tmp/paperless
+	local tmp_dir="/tmp/paperless"
+	echo "Creating directory ${tmp_dir}"
+	mkdir -p "${tmp_dir}"
 
 	set +e
 	echo "Adjusting permissions of paperless files. This may take a while."
-	chown -R paperless:paperless /tmp/paperless
-	find .. -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
+	chown -R paperless:paperless ${tmp_dir}
+	for dir in "${export_dir}" "${DATA_DIR}" "${MEDIA_ROOT_DIR}"; do
+		find "${dir}" -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
+	done
 	set -e
 
 	gosu paperless /sbin/docker-prepare.sh

docker/docker-prepare.sh

@@ -3,16 +3,17 @@
 set -e
 
 wait_for_postgres() {
-	attempt_num=1
-	max_attempts=5
+	local attempt_num=1
+	local max_attempts=5
 
 	echo "Waiting for PostgreSQL to start..."
 
-	host="${PAPERLESS_DBHOST:=localhost}"
-	port="${PAPERLESS_DBPORT:=5432}"
+	local host="${PAPERLESS_DBHOST:-localhost}"
+	local port="${PAPERLESS_DBPORT:-5432}"
 
-
-	while [ ! "$(pg_isready -h $host -p $port)" ]; do
+	# Disable warning, host and port can't have spaces
+	# shellcheck disable=SC2086
+	while [ ! "$(pg_isready -h ${host} -p ${port})" ]; do
 
 		if [ $attempt_num -eq $max_attempts ]; then
 			echo "Unable to connect to database."
@@ -43,17 +44,18 @@ migrations() {
 		flock 200
 		echo "Apply database migrations..."
 		python3 manage.py migrate
-	) 200>/usr/src/paperless/data/migration_lock
+	) 200>"${DATA_DIR}/migration_lock"
 }
 
 search_index() {
-	index_version=1
-	index_version_file=/usr/src/paperless/data/.index_version
 
-	if [[ (! -f "$index_version_file") || $(<$index_version_file) != "$index_version" ]]; then
+	local index_version=1
+	local index_version_file=${DATA_DIR}/.index_version
+
+	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then
 		echo "Search index out of date. Updating..."
 		python3 manage.py document_index reindex --no-progress-bar
-		echo $index_version | tee $index_version_file >/dev/null
+		echo ${index_version} | tee "${index_version_file}" >/dev/null
 	fi
 }
 

docker/install_management_commands.sh

@@ -2,7 +2,18 @@
 
 set -eu
 
-for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
+for command in decrypt_documents \
+	document_archiver \
+	document_exporter \
+	document_importer \
+	mail_fetcher \
+	document_create_classifier \
+	document_index \
+	document_renamer \
+	document_retagger \
+	document_thumbnails \
+	document_sanity_checker \
+	manage_superuser;
 do
 	echo "installing $command..."
 	sed "s/management_command/$command/g" management_script.sh > /usr/local/bin/$command

docker/wait-for-redis.py

@@ -26,9 +26,11 @@ if __name__ == "__main__":
             try:
                 client.ping()
                 break
-            except Exception:
+            except Exception as e:
                 print(
-                    f"Redis ping #{attempt} failed, waiting {RETRY_SLEEP_SECONDS}s",
+                    f"Redis ping #{attempt} failed.\n"
+                    f"Error: {str(e)}.\n"
+                    f"Waiting {RETRY_SLEEP_SECONDS}s",
                     flush=True,
                 )
                 time.sleep(RETRY_SLEEP_SECONDS)