Files
paperless-ngx/.github/workflows/ci.yml
2025-08-30 20:06:17 -07:00

637 lines
25 KiB
YAML

name: ci
on:
push:
tags:
# https://semver.org/#spec-item-2
- 'v[0-9]+.[0-9]+.[0-9]+'
# https://semver.org/#spec-item-9
- 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+'
branches-ignore:
- 'translations**'
pull_request:
branches-ignore:
- 'translations**'
env:
DEFAULT_UV_VERSION: "0.8.x"
# This is the default version of Python to use in most steps which aren't specific
DEFAULT_PYTHON_VERSION: "3.11"
NLTK_DATA: "/usr/share/nltk_data"
jobs:
# pre-commit:
# # We want to run on external PRs, but not on our own internal PRs as they'll be run
# # by the push to the branch. Without this if check, checks are duplicated since
# # internal PRs match both the push and pull_request events.
# if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
# name: Linting Checks
# runs-on: ubuntu-24.04
# steps:
# - name: Checkout repository
# uses: actions/checkout@v4
# - name: Install python
# uses: actions/setup-python@v5
# with:
# python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
# - name: Check files
# uses: pre-commit/action@v3.0.1
# documentation:
# name: "Build & Deploy Documentation"
# runs-on: ubuntu-24.04
# needs:
# - pre-commit
# steps:
# - name: Checkout
# uses: actions/checkout@v4
# - name: Set up Python
# id: setup-python
# uses: actions/setup-python@v5
# with:
# python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
# - name: Install uv
# uses: astral-sh/setup-uv@v6
# with:
# version: ${{ env.DEFAULT_UV_VERSION }}
# enable-cache: true
# python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
# - name: Install Python dependencies
# run: |
# uv sync --python ${{ steps.setup-python.outputs.python-version }} --dev --frozen
# - name: Make documentation
# run: |
# uv run \
# --python ${{ steps.setup-python.outputs.python-version }} \
# --dev \
# --frozen \
# mkdocs build --config-file ./mkdocs.yml
# - name: Deploy documentation
# if: github.event_name == 'push' && github.ref == 'refs/heads/main'
# run: |
# echo "docs.paperless-ngx.com" > "${{ github.workspace }}/docs/CNAME"
# git config --global user.name "${{ github.actor }}"
# git config --global user.email "${{ github.actor }}@users.noreply.github.com"
# uv run \
# --python ${{ steps.setup-python.outputs.python-version }} \
# --dev \
# --frozen \
# mkdocs gh-deploy --force --no-history
# - name: Upload artifact
# uses: actions/upload-artifact@v4
# with:
# name: documentation
# path: site/
# retention-days: 7
# tests-backend:
# name: "Backend Tests (Python ${{ matrix.python-version }})"
# runs-on: ubuntu-24.04
# needs:
# - pre-commit
# strategy:
# matrix:
# python-version: ['3.10', '3.11', '3.12']
# fail-fast: false
# steps:
# - name: Checkout
# uses: actions/checkout@v4
# - name: Start containers
# run: |
# docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml pull --quiet
# docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml up --detach
# - name: Set up Python
# id: setup-python
# uses: actions/setup-python@v5
# with:
# python-version: "${{ matrix.python-version }}"
# - name: Install uv
# uses: astral-sh/setup-uv@v6
# with:
# version: ${{ env.DEFAULT_UV_VERSION }}
# enable-cache: true
# python-version: ${{ steps.setup-python.outputs.python-version }}
# - name: Install system dependencies
# run: |
# sudo apt-get update -qq
# sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
# - name: Configure ImageMagick
# run: |
# sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml
# - name: Install Python dependencies
# run: |
# uv sync \
# --python ${{ steps.setup-python.outputs.python-version }} \
# --group testing \
# --frozen
# - name: List installed Python dependencies
# run: |
# uv pip list
# - name: Install or update NLTK dependencies
# run: uv run python -m nltk.downloader punkt punkt_tab snowball_data stopwords -d ${{ env.NLTK_DATA }}
# - name: Tests
# env:
# NLTK_DATA: ${{ env.NLTK_DATA }}
# PAPERLESS_CI_TEST: 1
# # Enable paperless_mail testing against real server
# PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
# PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
# PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
# run: |
# uv run \
# --python ${{ steps.setup-python.outputs.python-version }} \
# --dev \
# --frozen \
# pytest
# - name: Upload backend test results to Codecov
# if: always()
# uses: codecov/test-results-action@v1
# with:
# token: ${{ secrets.CODECOV_TOKEN }}
# flags: backend-python-${{ matrix.python-version }}
# files: junit.xml
# - name: Upload backend coverage to Codecov
# uses: codecov/codecov-action@v5
# with:
# token: ${{ secrets.CODECOV_TOKEN }}
# flags: backend-python-${{ matrix.python-version }}
# files: coverage.xml
# - name: Stop containers
# if: always()
# run: |
# docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml logs
# docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml down
# install-frontend-dependencies:
# name: "Install Frontend Dependencies"
# runs-on: ubuntu-24.04
# needs:
# - pre-commit
# steps:
# - uses: actions/checkout@v4
# - name: Install pnpm
# uses: pnpm/action-setup@v4
# with:
# version: 10
# - name: Use Node.js 20
# uses: actions/setup-node@v4
# with:
# node-version: 20.x
# cache: 'pnpm'
# cache-dependency-path: 'src-ui/pnpm-lock.yaml'
# - name: Cache frontend dependencies
# id: cache-frontend-deps
# uses: actions/cache@v4
# with:
# path: |
# ~/.pnpm-store
# ~/.cache
# key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
# - name: Install dependencies
# run: cd src-ui && pnpm install
# tests-frontend:
# name: "Frontend Unit Tests (Node ${{ matrix.node-version }} - ${{ matrix.shard-index }}/${{ matrix.shard-count }})"
# runs-on: ubuntu-24.04
# needs:
# - install-frontend-dependencies
# strategy:
# fail-fast: false
# matrix:
# node-version: [20.x]
# shard-index: [1, 2, 3, 4]
# shard-count: [4]
# steps:
# - uses: actions/checkout@v4
# - name: Install pnpm
# uses: pnpm/action-setup@v4
# with:
# version: 10
# - name: Use Node.js 20
# uses: actions/setup-node@v4
# with:
# node-version: 20.x
# cache: 'pnpm'
# cache-dependency-path: 'src-ui/pnpm-lock.yaml'
# - name: Cache frontend dependencies
# id: cache-frontend-deps
# uses: actions/cache@v4
# with:
# path: |
# ~/.pnpm-store
# ~/.cache
# key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
# - name: Re-link Angular cli
# run: cd src-ui && pnpm link @angular/cli
# - name: Linting checks
# run: cd src-ui && pnpm run lint
# - name: Run Jest unit tests
# run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
# - name: Upload frontend test results to Codecov
# uses: codecov/test-results-action@v1
# if: always()
# with:
# token: ${{ secrets.CODECOV_TOKEN }}
# flags: frontend-node-${{ matrix.node-version }}
# directory: src-ui/
# - name: Upload frontend coverage to Codecov
# uses: codecov/codecov-action@v5
# with:
# token: ${{ secrets.CODECOV_TOKEN }}
# flags: frontend-node-${{ matrix.node-version }}
# directory: src-ui/coverage/
# tests-frontend-e2e:
# name: "Frontend E2E Tests (Node ${{ matrix.node-version }} - ${{ matrix.shard-index }}/${{ matrix.shard-count }})"
# runs-on: ubuntu-24.04
# needs:
# - install-frontend-dependencies
# strategy:
# fail-fast: false
# matrix:
# node-version: [20.x]
# shard-index: [1, 2]
# shard-count: [2]
# steps:
# - uses: actions/checkout@v4
# - name: Install pnpm
# uses: pnpm/action-setup@v4
# with:
# version: 10
# - name: Use Node.js 20
# uses: actions/setup-node@v4
# with:
# node-version: 20.x
# cache: 'pnpm'
# cache-dependency-path: 'src-ui/pnpm-lock.yaml'
# - name: Cache frontend dependencies
# id: cache-frontend-deps
# uses: actions/cache@v4
# with:
# path: |
# ~/.pnpm-store
# ~/.cache
# key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
# - name: Re-link Angular cli
# run: cd src-ui && pnpm link @angular/cli
# - name: Cache Playwright browsers
# uses: actions/cache@v4
# with:
# path: ~/.cache/ms-playwright
# key: ${{ runner.os }}-playwright-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
# restore-keys: |
# ${{ runner.os }}-playwright-
# - name: Install Playwright system dependencies
# run: npx playwright install-deps
# - name: Install dependencies
# run: cd src-ui && pnpm install --no-frozen-lockfile
# - name: Install Playwright
# run: cd src-ui && pnpm exec playwright install
# - name: Run Playwright e2e tests
# run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
# frontend-bundle-analysis:
# name: "Frontend Bundle Analysis"
# runs-on: ubuntu-24.04
# needs:
# - tests-frontend
# - tests-frontend-e2e
# steps:
# - uses: actions/checkout@v4
# - name: Install pnpm
# uses: pnpm/action-setup@v4
# with:
# version: 10
# - name: Use Node.js 20
# uses: actions/setup-node@v4
# with:
# node-version: 20.x
# cache: 'pnpm'
# cache-dependency-path: 'src-ui/pnpm-lock.yaml'
# - name: Cache frontend dependencies
# id: cache-frontend-deps
# uses: actions/cache@v4
# with:
# path: |
# ~/.pnpm-store
# ~/.cache
# key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/package-lock.json') }}
# - name: Re-link Angular cli
# run: cd src-ui && pnpm link @angular/cli
# - name: Build frontend and upload analysis
# env:
# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# run: cd src-ui && pnpm run build --configuration=production
build-docker-image:
name: Build Docker image for ${{ github.ref_name }}
runs-on: ubuntu-24.04
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || startsWith(github.ref, 'refs/heads/fix-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/l10n_'))
concurrency:
group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
cancel-in-progress: true
# needs:
# - tests-backend
# - tests-frontend
# - tests-frontend-e2e
steps:
- name: Check pushing to Docker Hub
id: push-other-places
# Only push to Dockerhub from the main repo AND the ref is either:
# main
# dev
# beta
# a tag
# Otherwise forks would require a Docker Hub account and secrets setup
run: |
if [[ ${{ github.repository_owner }} == "paperless-ngx" && ( ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
echo "Enabling DockerHub image push"
echo "enable=true" >> $GITHUB_OUTPUT
else
echo "Not pushing to DockerHub"
echo "enable=false" >> $GITHUB_OUTPUT
fi
- name: Set ghcr repository name
id: set-ghcr-repository
run: |
ghcr_name=$(echo "${{ github.repository }}" | awk '{ print tolower($0) }')
echo "Name is ${ghcr_name}"
echo "ghcr-repository=${ghcr_name}" >> $GITHUB_OUTPUT
- name: Gather Docker metadata
id: docker-meta
uses: docker/metadata-action@v5
with:
images: |
ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}
name=paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }}
name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }}
tags: |
# Tag branches with branch name
type=ref,event=branch
# Process semver tags
# For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Checkout
uses: actions/checkout@v4
# If https://github.com/docker/buildx/issues/1044 is resolved,
# the append input with a native arm64 arch could be used to
# significantly speed up building
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: arm64
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v3
# Don't attempt to login if not pushing to Docker Hub
if: steps.push-other-places.outputs.enable == 'true'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Quay.io
uses: docker/login-action@v3
# Don't attempt to login if not pushing to Quay.io
if: steps.push-other-places.outputs.enable == 'true'
with:
registry: quay.io
username: ${{ secrets.QUAY_USERNAME }}
password: ${{ secrets.QUAY_ROBOT_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}
build-args: |
PNGX_TAG_VERSION=${{ steps.docker-meta.outputs.version }}
# Get cache layers from this branch, then dev
# This allows new branches to get at least some cache benefits, generally from dev
cache-from: |
type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:dev
cache-to: |
type=registry,mode=max,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
- name: Inspect image
run: |
docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
- name: Export frontend artifact from docker
run: |
docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
- name: Upload frontend artifact
uses: actions/upload-artifact@v4
with:
name: frontend-compiled
path: src/documents/static/frontend/
retention-days: 7
# build-release:
# name: "Build Release"
# needs:
# - build-docker-image
# - documentation
# runs-on: ubuntu-24.04
# steps:
# - name: Checkout
# uses: actions/checkout@v4
# - name: Set up Python
# id: setup-python
# uses: actions/setup-python@v5
# with:
# python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
# - name: Install uv
# uses: astral-sh/setup-uv@v6
# with:
# version: ${{ env.DEFAULT_UV_VERSION }}
# enable-cache: true
# python-version: ${{ steps.setup-python.outputs.python-version }}
# - name: Install Python dependencies
# run: |
# uv sync --python ${{ steps.setup-python.outputs.python-version }} --dev --frozen
# - name: Install system dependencies
# run: |
# sudo apt-get update -qq
# sudo apt-get install -qq --no-install-recommends gettext liblept5
# - name: Download frontend artifact
# uses: actions/download-artifact@v4
# with:
# name: frontend-compiled
# path: src/documents/static/frontend/
# - name: Download documentation artifact
# uses: actions/download-artifact@v4
# with:
# name: documentation
# path: docs/_build/html/
# - name: Generate requirements file
# run: |
# uv export --quiet --no-dev --all-extras --format requirements-txt --output-file requirements.txt
# - name: Compile messages
# run: |
# cd src/
# uv run \
# --python ${{ steps.setup-python.outputs.python-version }} \
# manage.py compilemessages
# - name: Collect static files
# run: |
# cd src/
# uv run \
# --python ${{ steps.setup-python.outputs.python-version }} \
# manage.py collectstatic --no-input
# - name: Move files
# run: |
# echo "Making dist folders"
# for directory in dist \
# dist/paperless-ngx \
# dist/paperless-ngx/scripts;
# do
# mkdir --verbose --parents ${directory}
# done
# echo "Copying basic files"
# for file_name in .dockerignore \
# .env \
# Dockerfile \
# pyproject.toml \
# uv.lock \
# requirements.txt \
# LICENSE \
# README.md \
# paperless.conf.example
# do
# cp --verbose ${file_name} dist/paperless-ngx/
# done
# mv --verbose dist/paperless-ngx/paperless.conf.example dist/paperless-ngx/paperless.conf
# echo "Copying Docker related files"
# cp --recursive docker/ dist/paperless-ngx/docker
# echo "Copying startup scripts"
# cp --verbose scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
# echo "Copying source files"
# cp --recursive src/ dist/paperless-ngx/src
# echo "Copying documentation"
# cp --recursive docs/_build/html/ dist/paperless-ngx/docs
# mv --verbose static dist/paperless-ngx
# - name: Make release package
# run: |
# echo "Creating release archive"
# cd dist
# sudo chown -R 1000:1000 paperless-ngx/
# tar -cJf paperless-ngx.tar.xz paperless-ngx/
# - name: Upload release artifact
# uses: actions/upload-artifact@v4
# with:
# name: release
# path: dist/paperless-ngx.tar.xz
# retention-days: 7
# publish-release:
# name: "Publish Release"
# runs-on: ubuntu-24.04
# outputs:
# prerelease: ${{ steps.get_version.outputs.prerelease }}
# changelog: ${{ steps.create-release.outputs.body }}
# version: ${{ steps.get_version.outputs.version }}
# needs:
# - build-release
# if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
# steps:
# - name: Download release artifact
# uses: actions/download-artifact@v4
# with:
# name: release
# path: ./
# - name: Get version
# id: get_version
# run: |
# echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT
# if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
# echo "prerelease=true" >> $GITHUB_OUTPUT
# else
# echo "prerelease=false" >> $GITHUB_OUTPUT
# fi
# - name: Create Release and Changelog
# id: create-release
# uses: release-drafter/release-drafter@v6
# with:
# name: Paperless-ngx ${{ steps.get_version.outputs.version }}
# tag: ${{ steps.get_version.outputs.version }}
# version: ${{ steps.get_version.outputs.version }}
# prerelease: ${{ steps.get_version.outputs.prerelease }}
# publish: true # ensures release is not marked as draft
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# - name: Upload release archive
# id: upload-release-asset
# uses: shogo82148/actions-upload-release-asset@v1
# with:
# github_token: ${{ secrets.GITHUB_TOKEN }}
# upload_url: ${{ steps.create-release.outputs.upload_url }}
# asset_path: ./paperless-ngx.tar.xz
# asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
# asset_content_type: application/x-xz
# append-changelog:
# name: "Append Changelog"
# runs-on: ubuntu-24.04
# needs:
# - publish-release
# if: needs.publish-release.outputs.prerelease == 'false'
# steps:
# - name: Checkout
# uses: actions/checkout@v4
# with:
# ref: main
# - name: Set up Python
# id: setup-python
# uses: actions/setup-python@v5
# with:
# python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
# - name: Install uv
# uses: astral-sh/setup-uv@v6
# with:
# version: ${{ env.DEFAULT_UV_VERSION }}
# enable-cache: true
# python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
# - name: Append Changelog to docs
# id: append-Changelog
# working-directory: docs
# run: |
# git branch ${{ needs.publish-release.outputs.version }}-changelog
# git checkout ${{ needs.publish-release.outputs.version }}-changelog
# echo -e "# Changelog\n\n${{ needs.publish-release.outputs.changelog }}\n" > changelog-new.md
# echo "Manually linking usernames"
# sed -i -r 's|@([a-zA-Z0-9_]+) \(\[#|[@\1](https://github.com/\1) ([#|g' changelog-new.md
# echo "Removing unneeded comment tags"
# sed -i -r 's|@<!---->|@|g' changelog-new.md
# CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
# echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
# mv changelog-new.md changelog.md
# uv run \
# --python ${{ steps.setup-python.outputs.python-version }} \
# --dev \
# pre-commit run --files changelog.md || true
# git config --global user.name "github-actions"
# git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
# git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA"
# git push origin ${{ needs.publish-release.outputs.version }}-changelog
# - name: Create Pull Request
# uses: actions/github-script@v7
# with:
# script: |
# const { repo, owner } = context.repo;
# const result = await github.rest.pulls.create({
# title: 'Documentation: Add ${{ needs.publish-release.outputs.version }} changelog',
# owner,
# repo,
# head: '${{ needs.publish-release.outputs.version }}-changelog',
# base: 'main',
# body: 'This PR is auto-generated by CI.'
# });
# github.rest.issues.addLabels({
# owner,
# repo,
# issue_number: result.data.number,
# labels: ['documentation', 'skip-changelog']
# });