From fcc9847bc39ea02e70fbcf7b11b0c1bae434df5f Mon Sep 17 00:00:00 2001 From: phail Date: Fri, 28 Jun 2024 00:13:55 +0200 Subject: [PATCH] Development: Add VS Code Devcontainer Configuration (#7041) --- .devcontainer/Dockerfile | 180 ++++++++++++++++++ .devcontainer/README.md | 117 ++++++++++++ .devcontainer/devcontainer.json | 16 ++ ...ocker-compose.devcontainer.sqlite-tika.yml | 84 ++++++++ .devcontainer/vscode/launch.json | 43 +++++ .devcontainer/vscode/settings.json | 11 ++ .devcontainer/vscode/tasks.json | 136 +++++++++++++ .gitignore | 2 + 8 files changed, 589 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/README.md create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.devcontainer.sqlite-tika.yml create mode 100644 .devcontainer/vscode/launch.json create mode 100644 .devcontainer/vscode/settings.json create mode 100644 .devcontainer/vscode/tasks.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..2aecc45a1 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,180 @@ +# syntax=docker/dockerfile:1 + +FROM --platform=$BUILDPLATFORM docker.io/node:20-bookworm-slim as main-app + +ARG DEBIAN_FRONTEND=noninteractive + +# Buildx provided, must be defined to use though +ARG TARGETARCH + +# Can be workflow provided, defaults set for manual building +ARG JBIG2ENC_VERSION=0.29 +ARG QPDF_VERSION=11.9.0 +ARG GS_VERSION=10.03.1 + +# Set Python environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + # Ignore warning from Whitenoise + PYTHONWARNINGS="ignore:::django.http.response:517" \ + PNGX_CONTAINERIZED=1 + +# +# Begin installation and configuration +# Order the steps below from least often changed to most +# + +# Packages need for running +ARG RUNTIME_PACKAGES="\ + # General utils + curl \ + # Docker specific + gosu \ + # Timezones support + tzdata \ + # fonts for text file thumbnail generation + fonts-liberation \ + gettext \ + ghostscript \ + gnupg \ + icc-profiles-free \ + imagemagick \ + # PostgreSQL + postgresql-client \ + # MySQL / MariaDB + mariadb-client \ + # OCRmyPDF dependencies + tesseract-ocr \ + tesseract-ocr-eng \ + tesseract-ocr-deu \ + tesseract-ocr-fra \ + tesseract-ocr-ita \ + tesseract-ocr-spa \ + unpaper \ + pngquant \ + jbig2dec \ + # lxml + libxml2 \ + libxslt1.1 \ + # itself + qpdf \ + # Mime type detection + file \ + libmagic1 \ + media-types \ + zlib1g \ + # Barcode splitter + libzbar0 \ + poppler-utils \ + htop \ + sudo" + +# Install basic runtime packages. +# These change very infrequently +RUN set -eux \ + echo "Installing system packages" \ + && apt-get update \ + && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} + +ARG PYTHON_PACKAGES="\ + python3 \ + python3-pip \ + python3-wheel \ + pipenv \ + ca-certificates" + +RUN set -eux \ + echo "Installing python packages" \ + && apt-get update \ + && apt-get install --yes --quiet ${PYTHON_PACKAGES} + +RUN set -eux \ + && echo "Installing pre-built updates" \ + && echo "Installing qpdf ${QPDF_VERSION}" \ + && curl --fail --silent --show-error --location \ + --output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + && curl --fail --silent --show-error --location \ + --output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + && dpkg --install ./libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + && dpkg --install ./qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + && echo "Installing Ghostscript ${GS_VERSION}" \ + && curl --fail --silent --show-error --location \ + --output libgs10_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \ + && curl --fail --silent --show-error --location \ + --output ghostscript_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/ghostscript_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \ + && curl --fail --silent --show-error --location \ + --output libgs10-common_${GS_VERSION}.dfsg-1_all.deb \ + https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10-common_${GS_VERSION}.dfsg-1_all.deb \ + && dpkg --install ./libgs10-common_${GS_VERSION}.dfsg-1_all.deb \ + && dpkg --install ./libgs10_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \ + && dpkg --install ./ghostscript_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \ + && echo "Installing jbig2enc" \ + && curl --fail --silent --show-error --location \ + --output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/jbig2enc-${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \ + && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb + +# setup docker-specific things +# These change sometimes, but rarely +WORKDIR /usr/src/paperless/src/docker/ + +COPY [ \ + "docker/imagemagick-policy.xml", \ + "./" \ +] + +RUN set -eux \ + && echo "Configuring ImageMagick" \ + && mv imagemagick-policy.xml /etc/ImageMagick-6/policy.xml + +# Packages needed only for building a few quick Python +# dependencies +ARG BUILD_PACKAGES="\ + build-essential \ + git \ + # https://www.psycopg.org/docs/install.html#prerequisites + libpq-dev \ + # https://github.com/PyMySQL/mysqlclient#linux + default-libmysqlclient-dev \ + pkg-config \ + pre-commit" + +# hadolint ignore=DL3042 +RUN --mount=type=cache,target=/root/.cache/pip/,id=pip-cache \ + set -eux \ + && echo "Installing build system packages" \ + && apt-get update \ + && apt-get install --yes --quiet ${BUILD_PACKAGES} + +RUN set -eux \ + && npm update npm -g + +# add users, setup scripts +# Mount the compiled frontend to expected location +RUN set -eux \ + && echo "Setting up user/group" \ + && groupmod --new-name paperless node \ + && usermod --login paperless --home /usr/src/paperless node \ + && usermod -s /bin/bash paperless \ + && echo "paperless ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ + && echo "Creating volume directories" \ + && mkdir --parents --verbose /usr/src/paperless/paperless-ngx/data \ + && mkdir --parents --verbose /usr/src/paperless/paperless-ngx/media \ + && mkdir --parents --verbose /usr/src/paperless/paperless-ngx/consume \ + && mkdir --parents --verbose /usr/src/paperless/paperless-ngx/export \ + && mkdir --parents --verbose /usr/src/paperless/paperless-ngx/.venv \ + && echo "Adjusting all permissions" \ + && chown --from root:root --changes --recursive paperless:paperless /usr/src/paperless +# && echo "Collecting static files" \ +# && gosu paperless python3 manage.py collectstatic --clear --no-input --link \ +# && gosu paperless python3 manage.py compilemessages + +VOLUME ["/usr/src/paperless/paperless-ngx/data", \ + "/usr/src/paperless/paperless-ngx/media", \ + "/usr/src/paperless/paperless-ngx/consume", \ + "/usr/src/paperless/paperless-ngx/export", \ + "/usr/src/paperless/paperless-ngx/.venv"] diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 000000000..3644d90c3 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,117 @@ +# Paperless NGX Development Environment + +## Overview + +Welcome to the Paperless NGX development environment! This setup uses VSCode DevContainers to provide a consistent and seamless development experience. + +### What are DevContainers? + +DevContainers are a feature in VSCode that allows you to develop within a Docker container. This ensures that your development environment is consistent across different machines and setups. By defining a containerized environment, you can eliminate the "works on my machine" problem. + +### Advantages of DevContainers + +- **Consistency**: Same environment for all developers. +- **Isolation**: Separate development environment from your local machine. +- **Reproducibility**: Easily recreate the environment on any machine. +- **Pre-configured Tools**: Include all necessary tools and dependencies in the container. + +## DevContainer Setup + +The DevContainer configuration provides up all the necessary services for Paperless NGX, including: + +- Redis +- Gotenberg +- Tika + +Data is stored using Docker volumes to ensure persistence across container restarts. + +## Configuration Files + +The setup includes debugging configurations (`launch.json`) and tasks (`tasks.json`) to help you manage and debug various parts of the project: + +- **Backend Debugging:** + - `manage.py runserver` + - `manage.py document-consumer` + - `celery` +- **Maintenance Tasks:** + - Create superuser + - Run migrations + - Recreate virtual environment (`.venv` with pipenv) + - Compile frontend assets + +## Getting Started + +### Step 1: Running the DevContainer + +To start the DevContainer: + +1. Open VSCode. +2. Open the project folder. +3. Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` +4. Type and select `Dev Containers: Rebuild and Reopen in Container`. + +VSCode will build and start the DevContainer environment. + +### Step 2: Initial Setup + +Once the DevContainer is up and running, perform the following steps: + +1. **Compile Frontend Assets**: + + - Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` + - Select `Tasks: Run Task`. + - Choose `Frontend Compile`. + +2. **Run Database Migrations**: + + - Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` + - Select `Tasks: Run Task`. + - Choose `Migrate Database`. + +3. **Create Superuser**: + - Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` + - Select `Tasks: Run Task`. + - Choose `Create Superuser`. + +### Debugging and Running Services + +You can start and debug backend services either as debugging sessions via `launch.json` or as tasks. + +#### Using `launch.json`: + +1. Press `F5` or go to the **Run and Debug** view in VSCode. +2. Select the desired configuration: + - `Runserver` + - `Document Consumer` + - `Celery` + +#### Using Tasks: + +1. Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` +2. Select `Tasks: Run Task`. +3. Choose the desired task: + - `Runserver` + - `Document Consumer` + - `Celery` + +### Additional Maintenance Tasks + +Additional tasks are available for common maintenance operations: + +- **Recreate .venv**: For setting up the virtual environment using pipenv. +- **Migrate Database**: To apply database migrations. +- **Create Superuser**: To create an admin user for the application. + +## Let's Get Started! + +Follow the steps above to get your development environment up and running. Happy coding! diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..e0fa56109 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,16 @@ +{ + "name": "Paperless Development", + "dockerComposeFile": "docker-compose.devcontainer.sqlite-tika.yml", + "service": "paperless-development", + "workspaceFolder": "/usr/src/paperless/paperless-ngx", + "postCreateCommand": "/bin/bash -c pre-commit install && pipenv install --dev", + "customizations": { + "vscode": { + "extensions": [ + "mhutchie.git-graph", + "ms-python.python" + ] + } + }, + "remoteUser": "paperless" + } diff --git a/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml b/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml new file mode 100644 index 000000000..cd978ebcb --- /dev/null +++ b/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml @@ -0,0 +1,84 @@ +# Docker Compose file for developing Paperless NGX in VSCode DevContainers. +# This file contains everything Paperless NGX needs to run. +# Paperless supports amd64, arm, and arm64 hardware. +# All compose files of Paperless configure it in the following way: +# +# - Paperless is (re)started on system boot if it was running before shutdown. +# - Docker volumes for storing data are managed by Docker. +# - Folders for importing and exporting files are created in the same directory +# as this file and mounted to the correct folders inside the container. +# - Paperless listens on port 8000. +# +# SQLite is used as the database. The SQLite file is stored in the data volume. +# +# In addition, this Docker Compose file adds the following optional +# configurations: +# +# - Apache Tika and Gotenberg servers are started with Paperless NGX and Paperless +# is configured to use these services. These provide support for consuming +# Office documents (Word, Excel, PowerPoint, and their LibreOffice counterparts). +# +# This file is intended only to be used through VSCOde devcontainers. See README.md +# in the folder .devcontainer. + + +services: + broker: + image: docker.io/library/redis:7 + restart: unless-stopped + volumes: + - redisdata:/data + + # No ports need to be exposed; the VSCode DevContainer plugin manages them. + paperless-development: + image: paperless-ngx + build: + context: ../ # Dockerfile cannot access files from parent directories if context is not set. + dockerfile: ./.devcontainer/Dockerfile + restart: unless-stopped + depends_on: + - broker + - gotenberg + - tika + volumes: + - ..:/usr/src/paperless/paperless-ngx:delegated + - ../.devcontainer/vscode:/usr/src/paperless/paperless-ngx/.vscode:delegated # VSCode config files + - pipenv:/usr/src/paperless/paperless-ngx/.venv # Pipenv environment persisted in volume + - /usr/src/paperless/paperless-ngx/src/documents/static/frontend # Static frontend files exist only in container + - /usr/src/paperless/paperless-ngx/src/.pytest_cache + - /usr/src/paperless/paperless-ngx/.ruff_cache + - /usr/src/paperless/paperless-ngx/htmlcov + - /usr/src/paperless/paperless-ngx/.coverage + - data:/usr/src/paperless/paperless-ngx/data + - media:/usr/src/paperless/paperless-ngx/media + environment: + PAPERLESS_REDIS: redis://broker:6379 + PAPERLESS_TIKA_ENABLED: 1 + PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000 + PAPERLESS_TIKA_ENDPOINT: http://tika:9998 + PAPERLESS_STATICDIR: ./src/documents/static + PAPERLESS_DEBUG: true + + # Overrides default command so things don't shut down after the process ends. + command: /bin/sh -c "chown -R paperless:paperless /usr/src/paperless/paperless-ngx/src/documents/static/frontend && chown -R paperless:paperless /usr/src/paperless/paperless-ngx/.ruff_cache && while sleep 1000; do :; done" + + gotenberg: + image: docker.io/gotenberg/gotenberg:7.10 + restart: unless-stopped + + # The Gotenberg Chromium route is used to convert .eml files. We do not + # want to allow external content like tracking pixels or even JavaScript. + command: + - "gotenberg" + - "--chromium-disable-javascript=true" + - "--chromium-allow-list=file:///tmp/.*" + + tika: + image: docker.io/apache/tika:latest + restart: unless-stopped + +volumes: + data: + media: + redisdata: + pipenv: diff --git a/.devcontainer/vscode/launch.json b/.devcontainer/vscode/launch.json new file mode 100644 index 000000000..2c50af44b --- /dev/null +++ b/.devcontainer/vscode/launch.json @@ -0,0 +1,43 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "manage.py runserver", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/src/manage.py", + "console": "integratedTerminal", + "justMyCode": true, + "args": ["runserver"], + "django": true + }, + { + "name": "manage.py document_consumer", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/src/manage.py", + "console": "integratedTerminal", + "justMyCode": true, + "args": ["document_consumer"], + "django": true + }, + { + "name": "celery", + "type": "python", + "cwd": "${workspaceFolder}/src", + "request": "launch", + "module": "celery", + "console": "integratedTerminal", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + }, + "args": [ + "-A", + "paperless", + "worker", + "-l", + "DEBUG" + ] + } + ] +} diff --git a/.devcontainer/vscode/settings.json b/.devcontainer/vscode/settings.json new file mode 100644 index 000000000..86c718cad --- /dev/null +++ b/.devcontainer/vscode/settings.json @@ -0,0 +1,11 @@ +{ + "python.testing.pytestArgs": [ + "src" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "files.watcherExclude": { + "**/.venv/**": true, + "**/pytest_cache/**": true + } +} diff --git a/.devcontainer/vscode/tasks.json b/.devcontainer/vscode/tasks.json new file mode 100644 index 000000000..fa27ba6bc --- /dev/null +++ b/.devcontainer/vscode/tasks.json @@ -0,0 +1,136 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "manage.py document_consumer", + "type": "shell", + "command": "pipenv run python manage.py document_consumer", + "group": "build", + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + + }, + { + "label": "manage.py runserver", + "type": "shell", + "command": "pipenv run python manage.py runserver", + "group": "build", + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + + }, + { + "label": "Maintenance: manage.py migrate", + "type": "shell", + "command": "pipenv run python manage.py migrate", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + }, + { + "label": "Maintenance: manage.py createsuperuser", + "type": "shell", + "command": "pipenv run python manage.py createsuperuser", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + }, + { + "label": "compile frontend", + "type": "shell", + "command": "npm ci && ./node_modules/.bin/ng build --configuration production", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src-ui" + } + }, + { + "label": "Maintenance: recreate .venv", + "type": "shell", + "command": "rm -R -v .venv/* || pipenv install --dev", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}" + } + }, + { + "label": "Celery Worker", + "type": "shell", + "command": "pipenv run celery --app paperless worker -l DEBUG", + "group": { + "kind": "build", + "isDefault": true + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + } + ] + } diff --git a/.gitignore b/.gitignore index f5bf2de6a..3351a924b 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,8 @@ target/ .vscode /src-ui/.vscode /docs/.vscode +.vscode-server +*CommandMarker # Other stuff that doesn't belong .virtualenv