Experiment with using uv, initial work

Fixes

Final pipenv things

Updates and fixes

Fixes locks

.github/workflows/ci.yml

@@ -14,9 +14,7 @@ on:
       - 'translations**'
 
 env:
-  # This is the version of pipenv all the steps will use
+  DEFAULT_UV_VERSION: "0.6.x"
-  # If changing this, change Dockerfile
-  DEFAULT_PIP_ENV_VERSION: "2024.4.1"
   # This is the default version of Python to use in most steps which aren't specific
   DEFAULT_PYTHON_VERSION: "3.11"
 
@@ -59,24 +57,24 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
       -
-        name: Install pipenv
+        name: Install uv
-        run: |
+        uses: astral-sh/setup-uv@v5
-          pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
+        with:
+          version: ${{ env.DEFAULT_UV_VERSION }}
+          enable-cache: true
       -
         name: Install dependencies
         run: |
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
+          uv sync --python ${{ steps.setup-python.outputs.python-version }} --dev --frozen
       -
         name: List installed Python dependencies
         run: |
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list
+          uv pip list
       -
         name: Make documentation
         run: |
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs build --config-file ./mkdocs.yml
+          uv run --python ${{ steps.setup-python.outputs.python-version }} mkdocs build --config-file ./mkdocs.yml
       -
         name: Deploy documentation
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -117,12 +115,12 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "${{ matrix.python-version }}"
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
       -
-        name: Install pipenv
+        name: Install uv
-        run: |
+        uses: astral-sh/setup-uv@v5
-          pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
+        with:
+          version: ${{ env.DEFAULT_UV_VERSION }}
+          enable-cache: true
       -
         name: Install system dependencies
         run: |
@@ -135,12 +133,11 @@ jobs:
       -
         name: Install Python dependencies
         run: |
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} run python --version
+          uv sync --python ${{ steps.setup-python.outputs.python-version }} --dev --frozen
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
       -
         name: List installed Python dependencies
         run: |
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list
+          uv pip list
       -
         name: Tests
         env:
@@ -151,7 +148,7 @@ jobs:
           PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
         run: |
           cd src/
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra
+          uv run --python ${{ steps.setup-python.outputs.python-version }} pytest -ra
       -
         name: Upload coverage
         if: ${{ matrix.python-version == env.DEFAULT_PYTHON_VERSION }}
@@ -472,16 +469,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
       -
-        name: Install pipenv + tools
+        name: Install uv
-        run: |
+        uses: astral-sh/setup-uv@v5
-          pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
+        with:
+          version: ${{ env.DEFAULT_UV_VERSION }}
+          enable-cache: true
       -
         name: Install Python dependencies
         run: |
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
+          uv sync --python ${{ steps.setup-python.outputs.python-version }} --dev --frozen
       -
         name: Install system dependencies
         run: |
@@ -502,17 +499,17 @@ jobs:
       -
         name: Generate requirements file
         run: |
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} requirements > requirements.txt
+           uv export --no-dev --frozen --format requirements-txt
       -
         name: Compile messages
         run: |
           cd src/
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py compilemessages
+          uv run --python ${{ steps.setup-python.outputs.python-version }} manage.py compilemessages
       -
         name: Collect static files
         run: |
           cd src/
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py collectstatic --no-input
+          uv run --python ${{ steps.setup-python.outputs.python-version }} manage.py collectstatic --no-input
       -
         name: Move files
         run: |
@@ -631,15 +628,16 @@ jobs:
           ref: main
       -
         name: Set up Python
+        id: setup-python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
       -
-        name: Install pipenv + tools
+        name: Install uv
-        run: |
+        uses: astral-sh/setup-uv@v5
-          pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
+        with:
+          version: ${{ env.DEFAULT_UV_VERSION }}
+          enable-cache: true
       -
         name: Append Changelog to docs
         id: append-Changelog
@@ -655,7 +653,7 @@ jobs:
           CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
           echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
           mv changelog-new.md changelog.md
-          pipenv run pre-commit run --files changelog.md || true
+          uv run --python ${{ steps.setup-python.outputs.python-version }} pre-commit run --files changelog.md || true
           git config --global user.name "github-actions"
           git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA"

.pre-commit-config.yaml

@@ -55,6 +55,10 @@ repos:
     hooks:
       - id: ruff
       - id: ruff-format
+  - repo: https://github.com/tox-dev/pyproject-fmt
+    rev: "2.0.4"
+    hooks:
+      - id: pyproject-fmt
   # Dockerfile hooks
   - repo: https://github.com/AleksaC/hadolint-py
     rev: v2.12.0.3

Dockerfile

@@ -26,28 +26,11 @@ esac
 RUN set -eux \
   && ./node_modules/.bin/ng build --configuration production
 
-# Stage: pipenv-base
-# Purpose: Generates a requirements.txt file for building
-# Comments:
-#  - pipenv dependencies are not left in the final image
-#  - pipenv can't touch the final image somehow
-FROM --platform=$BUILDPLATFORM docker.io/python:3.12-alpine AS pipenv-base
-
-WORKDIR /usr/src/pipenv
-
-COPY Pipfile* ./
-
-RUN set -eux \
-  && echo "Installing pipenv" \
-    && python3 -m pip install --no-cache-dir --upgrade pipenv==2024.4.1 \
-  && echo "Generating requirement.txt" \
-    && pipenv requirements > requirements.txt
-
 # Stage: s6-overlay-base
 # Purpose: Installs s6-overlay and rootfs
 # Comments:
 #  - Don't leave anything extra in here either
-FROM docker.io/python:3.12-slim-bookworm AS s6-overlay-base
+FROM ghcr.io/astral-sh/uv:0.6.3-python3.12-bookworm-slim AS s6-overlay-base
 
 WORKDIR /usr/src/s6
 
@@ -56,6 +39,7 @@ ENV \
     S6_BEHAVIOUR_IF_STAGE2_FAILS=2 \
     S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0 \
     S6_VERBOSITY=1 \
+    UV_LINK_MODE=copy \
     PATH=/command:$PATH
 
 # Buildx provided, must be defined to use though
@@ -125,7 +109,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     # Ignore warning from Whitenoise
     PYTHONWARNINGS="ignore:::django.http.response:517" \
-    PNGX_CONTAINERIZED=1
+    PNGX_CONTAINERIZED=1 \
+    PATH="/usr/src/paperless/src/.venv/bin:$PATH"
 
 #
 # Begin installation and configuration
@@ -213,7 +198,7 @@ WORKDIR /usr/src/paperless/src/
 
 # Python dependencies
 # Change pretty frequently
-COPY --chown=1000:1000 --from=pipenv-base /usr/src/pipenv/requirements.txt ./
+COPY --chown=1000:1000 ["pyproject.toml", "uv.lock", "/usr/src/paperless/src/"]
 
 # Packages needed only for building a few quick Python
 # dependencies
@@ -230,19 +215,19 @@ ARG ZXING_VERSION=2.3.0
 ARG PSYCOPG_VERSION=3.2.4
 
 # hadolint ignore=DL3042
-RUN --mount=type=cache,target=/root/.cache/pip/,id=pip-cache \
+RUN --mount=type=cache,target=/root/.cache/uv,id=pip-cache \
   set -eux \
   && echo "Installing build system packages" \
     && apt-get update \
     && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
-    && python3 -m pip install --upgrade wheel \
   && echo "Installing Python requirements" \
     && curl --fail --silent --no-progress-meter --show-error --location --remote-name-all --parallel --parallel-max 4 \
       https://github.com/paperless-ngx/builder/releases/download/psycopg-${PSYCOPG_VERSION}/psycopg_c-${PSYCOPG_VERSION}-cp312-cp312-linux_x86_64.whl \
       https://github.com/paperless-ngx/builder/releases/download/psycopg-${PSYCOPG_VERSION}/psycopg_c-${PSYCOPG_VERSION}-cp312-cp312-linux_aarch64.whl \
       https://github.com/paperless-ngx/builder/releases/download/zxing-${ZXING_VERSION}/zxing_cpp-${ZXING_VERSION}-cp312-cp312-linux_aarch64.whl \
       https://github.com/paperless-ngx/builder/releases/download/zxing-${ZXING_VERSION}/zxing_cpp-${ZXING_VERSION}-cp312-cp312-linux_x86_64.whl \
-    && python3 -m pip install --default-timeout=1000 --find-links . --requirement requirements.txt \
+    && uv sync --no-progress --frozen --no-dev --no-python-downloads --python-preference system --find-links . \
+    && chown -R 1000:1000 . \
   && echo "Installing NLTK data" \
     && python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" snowball_data \
     && python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" stopwords \

pyproject.toml

@@ -0,0 +1,97 @@
+[project]
+name = "paperless-ngx"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+  "bleach~=6.1.0",
+  "celery[redis]~=5.4.0",
+  "channels~=4.1",
+  "channels-redis~=4.2",
+  "concurrent-log-handler~=0.9.25",
+  "dateparser~=1.2",
+  # WARNING: django does not use semver.
+  #          Only patch versions are guaranteed to not introduce breaking changes.
+  "django~=5.1.1",
+  "django-allauth[socialaccount,mfa]~=65.0.2",
+  "django-auditlog~=3.0.0",
+  "django-celery-results~=2.5.1",
+  "django-compression-middleware~=0.5.0",
+  "django-cors-headers~=4.4.0",
+  "django-extensions~=3.2.3",
+  "django-filter~=24.3",
+  "django-guardian~=2.4.0",
+  "django-multiselectfield~=0.1.13",
+  "django-soft-delete~=1.0.15",
+  "djangorestframework~=3.15",
+  "djangorestframework-guardian~=0.3.0",
+  "drf-writable-nested~=0.7.0",
+  "filelock~=3.16.1",
+  "flower~=2.0.1",
+  "gotenberg-client~=0.6.0",
+  "imap-tools~=1.7.3",
+  "inotifyrecursive~=0.3",
+  "langdetect~=1.0.9",
+  "mysqlclient~=2.2.4",
+  "nltk~=3.9.1",
+  "ocrmypdf~=16.5.0",
+  "pathvalidate~=3.2.1",
+  "pdf2image~=1.17.0",
+  "psycopg[c]==3.2.2",
+  "python-dateutil~=2.9.0",
+  "python-dotenv~=1.0.1",
+  "python-gnupg~=0.5.3",
+  "python-ipware~=3.0.0",
+  "python-magic~=0.4.27",
+  "pyzbar~=0.1.9",
+  "rapidfuzz~=3.10.0",
+  "redis[hiredis]~=5.1.1",
+  "scikit-learn~=1.5.2",
+  "setproctitle~=1.3.3",
+  "tika-client~=0.6.0",
+  "tqdm~=4.66.5",
+  "watchdog~=4.0",
+  "whitenoise~=6.7",
+  "whoosh~=2.7",
+  "zxing-cpp~=2.2.0; platform_machine=='x86_64'",
+  "jinja2>=3.1.4",
+  "drf_spectacular>=0.28.0",
+  "drf-spectacular-sidecar>=2025.2.1",
+  "granian>=1.7.6",
+]
+
+[tool.uv]
+package = false
+dev-dependencies = [
+  # Linting
+  "pre-commit~=4.0.0",
+  "pre-commit-uv~=4.1.3",
+  "ruff~=0.6.9",
+  # Testing
+  "factory-boy~=3.3.1",
+  "pytest~=8.3.3",
+  "pytest-cov~=5.0.0",
+  "pytest-django~=4.9.0",
+  "pytest-httpx",
+  "pytest-env",
+  "pytest-sugar",
+  "pytest-xdist",
+  "pytest-mock",
+  "pytest-rerunfailures",
+  "imagehash",
+  "daphne",
+  # Documentation
+  "mkdocs-material",
+  "mkdocs-glightbox",
+]
+environments = [
+  "sys_platform == 'darwin'",
+  "sys_platform == 'linux'",
+]