diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 777f833da..4ea22f997 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -76,18 +76,15 @@ RUN set -eux \ && apt-get update \ && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} -ARG PYTHON_PACKAGES="\ - python3 \ - python3-pip \ - python3-wheel \ - pipenv \ - ca-certificates" +ARG PYTHON_PACKAGES="ca-certificates" RUN set -eux \ echo "Installing python packages" \ && apt-get update \ && apt-get install --yes --quiet ${PYTHON_PACKAGES} +COPY --from=ghcr.io/astral-sh/uv:0.4 /uv /bin/uv + RUN set -eux \ && echo "Installing pre-built updates" \ && echo "Installing qpdf ${QPDF_VERSION}" \ @@ -131,6 +128,8 @@ RUN set -eux \ && echo "Configuring ImageMagick" \ && mv paperless-policy.xml /etc/ImageMagick-6/policy.xml +COPY --from=ghcr.io/astral-sh/uv:0.4 /uv /bin/uv + # Packages needed only for building a few quick Python # dependencies ARG BUILD_PACKAGES="\ @@ -140,11 +139,10 @@ ARG BUILD_PACKAGES="\ libpq-dev \ # https://github.com/PyMySQL/mysqlclient#linux default-libmysqlclient-dev \ - pkg-config \ - pre-commit" + pkg-config" # hadolint ignore=DL3042 -RUN --mount=type=cache,target=/root/.cache/pip/,id=pip-cache \ +RUN --mount=type=cache,target=/root/.cache/uv,id=pip-cache \ set -eux \ && echo "Installing build system packages" \ && apt-get update \ @@ -169,9 +167,6 @@ RUN set -eux \ && mkdir --parents --verbose /usr/src/paperless/paperless-ngx/.venv \ && echo "Adjusting all permissions" \ && chown --from root:root --changes --recursive paperless:paperless /usr/src/paperless -# && echo "Collecting static files" \ -# && gosu paperless python3 manage.py collectstatic --clear --no-input --link \ -# && gosu paperless python3 manage.py compilemessages VOLUME ["/usr/src/paperless/paperless-ngx/data", \ "/usr/src/paperless/paperless-ngx/media", \ diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 000000000..15866174b --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,117 @@ +# Paperless NGX Development Environment + +## Overview + +Welcome to the Paperless NGX development environment! This setup uses VSCode DevContainers to provide a consistent and seamless development experience. + +### What are DevContainers? + +DevContainers are a feature in VSCode that allows you to develop within a Docker container. This ensures that your development environment is consistent across different machines and setups. By defining a containerized environment, you can eliminate the "works on my machine" problem. + +### Advantages of DevContainers + +- **Consistency**: Same environment for all developers. +- **Isolation**: Separate development environment from your local machine. +- **Reproducibility**: Easily recreate the environment on any machine. +- **Pre-configured Tools**: Include all necessary tools and dependencies in the container. + +## DevContainer Setup + +The DevContainer configuration provides up all the necessary services for Paperless NGX, including: + +- Redis +- Gotenberg +- Tika + +Data is stored using Docker volumes to ensure persistence across container restarts. + +## Configuration Files + +The setup includes debugging configurations (`launch.json`) and tasks (`tasks.json`) to help you manage and debug various parts of the project: + +- **Backend Debugging:** + - `manage.py runserver` + - `manage.py document-consumer` + - `celery` +- **Maintenance Tasks:** + - Create superuser + - Run migrations + - Recreate virtual environment (`.venv` with `uv`) + - Compile frontend assets + +## Getting Started + +### Step 1: Running the DevContainer + +To start the DevContainer: + +1. Open VSCode. +2. Open the project folder. +3. Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` +4. Type and select `Dev Containers: Rebuild and Reopen in Container`. + +VSCode will build and start the DevContainer environment. + +### Step 2: Initial Setup + +Once the DevContainer is up and running, perform the following steps: + +1. **Compile Frontend Assets**: + + - Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` + - Select `Tasks: Run Task`. + - Choose `Frontend Compile`. + +2. **Run Database Migrations**: + + - Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` + - Select `Tasks: Run Task`. + - Choose `Migrate Database`. + +3. **Create Superuser**: + - Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` + - Select `Tasks: Run Task`. + - Choose `Create Superuser`. + +### Debugging and Running Services + +You can start and debug backend services either as debugging sessions via `launch.json` or as tasks. + +#### Using `launch.json`: + +1. Press `F5` or go to the **Run and Debug** view in VSCode. +2. Select the desired configuration: + - `Runserver` + - `Document Consumer` + - `Celery` + +#### Using Tasks: + +1. Open the command palette: + - **Windows/Linux**: `Ctrl+Shift+P` + - **Mac**: `Cmd+Shift+P` +2. Select `Tasks: Run Task`. +3. Choose the desired task: + - `Runserver` + - `Document Consumer` + - `Celery` + +### Additional Maintenance Tasks + +Additional tasks are available for common maintenance operations: + +- **Recreate .venv**: For setting up the virtual environment using `uv`. +- **Migrate Database**: To apply database migrations. +- **Create Superuser**: To create an admin user for the application. + +## Let's Get Started! + +Follow the steps above to get your development environment up and running. Happy coding! diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 34446caea..8017ac5bd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ "dockerComposeFile": "docker-compose.devcontainer.sqlite-tika.yml", "service": "paperless-development", "workspaceFolder": "/usr/src/paperless/paperless-ngx", - "postCreateCommand": "pipenv install --dev && pipenv run pre-commit install", + "postCreateCommand": "/bin/bash -c uv sync --dev && uv run pre-commit install", "customizations": { "vscode": { "extensions": [ diff --git a/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml b/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml index fa463da5c..8aead7fe2 100644 --- a/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml +++ b/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml @@ -43,7 +43,7 @@ services: volumes: - ..:/usr/src/paperless/paperless-ngx:delegated - ../.devcontainer/vscode:/usr/src/paperless/paperless-ngx/.vscode:delegated # VSCode config files - - pipenv:/usr/src/paperless/paperless-ngx/.venv + - virtualenv:/usr/src/paperless/paperless-ngx/.venv # Virtual environment persisted in volume - /usr/src/paperless/paperless-ngx/src/documents/static/frontend # Static frontend files exist only in container - /usr/src/paperless/paperless-ngx/src/.pytest_cache - /usr/src/paperless/paperless-ngx/.ruff_cache @@ -80,4 +80,7 @@ services: restart: unless-stopped volumes: - pipenv: + data: + media: + redisdata: + virtualenv: diff --git a/.devcontainer/vscode/tasks.json b/.devcontainer/vscode/tasks.json index 6c58cdaa3..3d89be884 100644 --- a/.devcontainer/vscode/tasks.json +++ b/.devcontainer/vscode/tasks.json @@ -5,7 +5,7 @@ "label": "Start: Celery Worker", "description": "Start the Celery Worker which processes background and consume tasks", "type": "shell", - "command": "pipenv run celery --app paperless worker -l DEBUG", + "command": "uv run celery --app paperless worker -l DEBUG", "isBackground": true, "options": { "cwd": "${workspaceFolder}/src" @@ -61,7 +61,7 @@ "label": "Start: Consumer Service (manage.py document_consumer)", "description": "Start the Consumer Service which processes files from a directory", "type": "shell", - "command": "pipenv run python manage.py document_consumer", + "command": "uv run python manage.py document_consumer", "group": "build", "presentation": { "echo": true, @@ -80,7 +80,7 @@ "label": "Start: Backend Server (manage.py runserver)", "description": "Start the Backend Server which serves the Django API and the compiled Angular frontend", "type": "shell", - "command": "pipenv run python manage.py runserver", + "command": "uv run python manage.py runserver", "group": "build", "presentation": { "echo": true, @@ -99,7 +99,7 @@ "label": "Maintenance: manage.py migrate", "description": "Apply database migrations", "type": "shell", - "command": "pipenv run python manage.py migrate", + "command": "uv run python manage.py migrate", "group": "none", "presentation": { "echo": true, @@ -118,7 +118,7 @@ "label": "Maintenance: Build Documentation", "description": "Build the documentation with MkDocs", "type": "shell", - "command": "pipenv run mkdocs build --config-file mkdocs.yml && pipenv run mkdocs serve", + "command": "uv run mkdocs build --config-file mkdocs.yml && uv run mkdocs serve", "group": "none", "presentation": { "echo": true, @@ -137,7 +137,7 @@ "label": "Maintenance: manage.py createsuperuser", "description": "Create a superuser", "type": "shell", - "command": "pipenv run python manage.py createsuperuser", + "command": "uv run python manage.py createsuperuser", "group": "none", "presentation": { "echo": true, @@ -156,7 +156,7 @@ "label": "Maintenance: recreate .venv", "description": "Recreate the python virtual environment and install python dependencies", "type": "shell", - "command": "rm -R -v .venv/* || pipenv install --dev", + "command": "rm -R -v .venv/* || uv install --dev", "group": "none", "presentation": { "echo": true, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 978f21fca..98309f3f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: echo "docs.paperless-ngx.com" > "${{ github.workspace }}/docs/CNAME" git config --global user.name "${{ github.actor }}" git config --global user.email "${{ github.actor }}@users.noreply.github.com" - pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs gh-deploy --force --no-history + uv run --python ${{ steps.setup-python.outputs.python-version }} mkdocs gh-deploy --force --no-history - name: Upload artifact uses: actions/upload-artifact@v4 @@ -525,8 +525,8 @@ jobs: for file_name in .dockerignore \ .env \ Dockerfile \ - Pipfile \ - Pipfile.lock \ + pyproject.toml \ + uv.lock \ requirements.txt \ LICENSE \ README.md \ diff --git a/docs/development.md b/docs/development.md index 11223d9f3..3be3b43c3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -60,7 +60,7 @@ first-time setup. Every command is executed directly from the root folder of the project unless specified otherwise. -1. Install prerequisites + pipenv as mentioned in +1. Install prerequisites + [uv](https://github.com/astral-sh/uv) as mentioned in [Bare metal route](setup.md#bare_metal). 2. Copy `paperless.conf.example` to `paperless.conf` and enable debug @@ -75,17 +75,13 @@ first-time setup. 4. Install the Python dependencies: ```bash - pipenv install --dev + $ uv sync --dev ``` - !!! note - - Using a virtual environment is highly recommended. You can spawn one via `pipenv shell`. - 5. Install pre-commit hooks: ```bash - pre-commit install + $ uv run pre-commit install ``` 6. Apply migrations and create a superuser for your development instance: @@ -93,8 +89,8 @@ first-time setup. ```bash # src/ - python3 manage.py migrate - python3 manage.py createsuperuser + $ uv run manage.py migrate + $ uv run manage.py createsuperuser ``` 7. You can now either ... @@ -332,9 +328,10 @@ LANGUAGES = [ The documentation is built using material-mkdocs, see their [documentation](https://squidfunk.github.io/mkdocs-material/reference/). If you want to build the documentation locally, this is how you do it: -1. Have an active pipenv shell (`pipenv shell`) and install Python dependencies: +1. Build the documentation ```bash +<<<<<<< HEAD pipenv install --dev ``` @@ -342,17 +339,20 @@ If you want to build the documentation locally, this is how you do it: ```bash mkdocs build --config-file mkdocs.yml +======= + $ uv run mkdocs build --config-file mkdocs.yml +>>>>>>> a1d6df0dc (Further purging of pipenv) ``` _alternatively..._ -3. Serve the documentation. This will spin up a +2. Serve the documentation. This will spin up a copy of the documentation at http://127.0.0.1:8000 that will automatically refresh every time you change something. ```bash - mkdocs serve + $ uv run mkdocs serve ``` ## Building the Docker image diff --git a/pyproject.toml b/pyproject.toml index 57df5ebbe..72b094a26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "gotenberg-client~=0.6.0", "imap-tools~=1.7.3", "inotifyrecursive~=0.3", + "jinja2>=3.1.4", "langdetect~=1.0.9", "mysqlclient~=2.2.4", "nltk~=3.9.1", @@ -60,7 +61,7 @@ dependencies = [ "watchdog~=4.0", "whitenoise~=6.7", "whoosh~=2.7", - "zxing-cpp~=2.2.0; platform_machine=='x86_64'", + "zxing-cpp~=2.2.0", "jinja2>=3.1.4", "drf_spectacular>=0.28.0", "drf-spectacular-sidecar>=2025.2.1",