mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-04-02 13:45:10 -05:00
530 lines
18 KiB
YAML
530 lines
18 KiB
YAML
name: ci
|
|
|
|
on:
|
|
push:
|
|
tags:
|
|
# https://semver.org/#spec-item-2
|
|
- 'v[0-9]+.[0-9]+.[0-9]+'
|
|
# https://semver.org/#spec-item-9
|
|
- 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+'
|
|
branches-ignore:
|
|
- 'translations**'
|
|
pull_request:
|
|
branches-ignore:
|
|
- 'translations**'
|
|
|
|
jobs:
|
|
pre-commit:
|
|
name: Linting Checks
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
-
|
|
name: Checkout repository
|
|
uses: actions/checkout@v3
|
|
|
|
-
|
|
name: Install tools
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.9"
|
|
|
|
-
|
|
name: Check files
|
|
uses: pre-commit/action@v3.0.0
|
|
|
|
documentation:
|
|
name: "Build Documentation"
|
|
runs-on: ubuntu-20.04
|
|
needs:
|
|
- pre-commit
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Deploy docs
|
|
uses: mhausenblas/mkdocs-deploy-gh-pages@master
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
CUSTOM_DOMAIN: paperless-ngx.com
|
|
CONFIG_FILE: mkdocs.yml
|
|
EXTRA_PACKAGES: build-base
|
|
|
|
tests-backend:
|
|
name: "Tests (${{ matrix.python-version }})"
|
|
runs-on: ubuntu-20.04
|
|
needs:
|
|
- pre-commit
|
|
strategy:
|
|
matrix:
|
|
python-version: ['3.8', '3.9', '3.10']
|
|
fail-fast: false
|
|
env:
|
|
# Enable Tika end to end testing
|
|
TIKA_LIVE: 1
|
|
# Enable paperless_mail testing against real server
|
|
PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
|
|
PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
|
|
PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
with:
|
|
fetch-depth: 0
|
|
-
|
|
name: Start containers
|
|
run: |
|
|
docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml pull --quiet
|
|
docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml up --detach
|
|
-
|
|
name: Install pipenv
|
|
run: |
|
|
pipx install pipenv==2022.10.12
|
|
-
|
|
name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "${{ matrix.python-version }}"
|
|
cache: "pipenv"
|
|
cache-dependency-path: 'Pipfile.lock'
|
|
-
|
|
name: Install system dependencies
|
|
run: |
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
|
|
-
|
|
name: Install Python dependencies
|
|
run: |
|
|
pipenv sync --dev
|
|
-
|
|
name: List installed Python dependencies
|
|
run: |
|
|
pipenv run pip list
|
|
-
|
|
name: Tests
|
|
run: |
|
|
cd src/
|
|
pipenv run pytest -rfEp
|
|
-
|
|
name: Get changed files
|
|
id: changed-files-specific
|
|
uses: tj-actions/changed-files@v34
|
|
with:
|
|
files: |
|
|
src/**
|
|
-
|
|
name: List all changed files
|
|
run: |
|
|
for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
|
|
echo "${file} was changed"
|
|
done
|
|
-
|
|
name: Publish coverage results
|
|
if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
# https://github.com/coveralls-clients/coveralls-python/issues/251
|
|
run: |
|
|
cd src/
|
|
pipenv run coveralls --service=github
|
|
-
|
|
name: Stop containers
|
|
if: always()
|
|
run: |
|
|
docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml logs
|
|
docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml down
|
|
|
|
tests-frontend:
|
|
name: "Tests Frontend"
|
|
runs-on: ubuntu-20.04
|
|
needs:
|
|
- pre-commit
|
|
strategy:
|
|
matrix:
|
|
node-version: [16.x]
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
-
|
|
name: Use Node.js ${{ matrix.node-version }}
|
|
uses: actions/setup-node@v3
|
|
with:
|
|
node-version: ${{ matrix.node-version }}
|
|
- run: cd src-ui && npm ci
|
|
- run: cd src-ui && npm run test
|
|
- run: cd src-ui && npm run e2e:ci
|
|
|
|
prepare-docker-build:
|
|
name: Prepare Docker Pipeline Data
|
|
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
|
|
runs-on: ubuntu-20.04
|
|
# If the push triggered the installer library workflow, wait for it to
|
|
# complete here. This ensures the required versions for the final
|
|
# image have been built, while not waiting at all if the versions haven't changed
|
|
concurrency:
|
|
group: build-installer-library
|
|
cancel-in-progress: false
|
|
needs:
|
|
- documentation
|
|
- tests-backend
|
|
- tests-frontend
|
|
steps:
|
|
-
|
|
name: Set ghcr repository name
|
|
id: set-ghcr-repository
|
|
run: |
|
|
ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
|
|
echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.9"
|
|
-
|
|
name: Setup qpdf image
|
|
id: qpdf-setup
|
|
run: |
|
|
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
|
|
|
|
echo ${build_json}
|
|
|
|
echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
|
|
-
|
|
name: Setup psycopg2 image
|
|
id: psycopg2-setup
|
|
run: |
|
|
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
|
|
|
|
echo ${build_json}
|
|
|
|
echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
|
|
-
|
|
name: Setup pikepdf image
|
|
id: pikepdf-setup
|
|
run: |
|
|
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
|
|
|
|
echo ${build_json}
|
|
|
|
echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
|
|
-
|
|
name: Setup jbig2enc image
|
|
id: jbig2enc-setup
|
|
run: |
|
|
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
|
|
|
|
echo ${build_json}
|
|
|
|
echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
|
|
|
|
outputs:
|
|
|
|
ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
|
|
|
|
qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
|
|
|
|
pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
|
|
|
|
psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
|
|
|
|
jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
|
|
|
|
# build and push image to docker hub.
|
|
build-docker-image:
|
|
runs-on: ubuntu-20.04
|
|
concurrency:
|
|
group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
|
|
cancel-in-progress: true
|
|
needs:
|
|
- prepare-docker-build
|
|
steps:
|
|
-
|
|
name: Check pushing to Docker Hub
|
|
id: docker-hub
|
|
# Only push to Dockerhub from the main repo AND the ref is either:
|
|
# main
|
|
# dev
|
|
# beta
|
|
# a tag
|
|
# Otherwise forks would require a Docker Hub account and secrets setup
|
|
run: |
|
|
if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
|
|
echo "Enabling DockerHub image push"
|
|
echo "enable=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "Not pushing to DockerHub"
|
|
echo "enable=false" >> $GITHUB_OUTPUT
|
|
fi
|
|
-
|
|
name: Gather Docker metadata
|
|
id: docker-meta
|
|
uses: docker/metadata-action@v4
|
|
with:
|
|
images: |
|
|
ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}
|
|
name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
|
|
tags: |
|
|
# Tag branches with branch name
|
|
type=ref,event=branch
|
|
# Process semver tags
|
|
# For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag
|
|
type=semver,pattern={{version}}
|
|
type=semver,pattern={{major}}.{{minor}}
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v2
|
|
-
|
|
name: Set up QEMU
|
|
uses: docker/setup-qemu-action@v2
|
|
-
|
|
name: Login to Github Container Registry
|
|
uses: docker/login-action@v2
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
-
|
|
name: Login to Docker Hub
|
|
uses: docker/login-action@v2
|
|
# Don't attempt to login is not pushing to Docker Hub
|
|
if: steps.docker-hub.outputs.enable == 'true'
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
-
|
|
name: Build and push
|
|
uses: docker/build-push-action@v3
|
|
with:
|
|
context: .
|
|
file: ./Dockerfile
|
|
platforms: linux/amd64,linux/arm/v7,linux/arm64
|
|
push: ${{ github.event_name != 'pull_request' }}
|
|
tags: ${{ steps.docker-meta.outputs.tags }}
|
|
labels: ${{ steps.docker-meta.outputs.labels }}
|
|
build-args: |
|
|
JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
|
|
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
|
|
PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
|
|
PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
|
|
# Get cache layers from this branch, then dev, then main
|
|
# This allows new branches to get at least some cache benefits, generally from dev
|
|
cache-from: |
|
|
type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
|
|
type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:dev
|
|
type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:main
|
|
cache-to: |
|
|
type=registry,mode=max,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
|
|
-
|
|
name: Inspect image
|
|
run: |
|
|
docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
|
|
-
|
|
name: Export frontend artifact from docker
|
|
run: |
|
|
docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
|
|
docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
|
|
-
|
|
name: Upload frontend artifact
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: frontend-compiled
|
|
path: src/documents/static/frontend/
|
|
|
|
build-release:
|
|
needs:
|
|
- build-docker-image
|
|
runs-on: ubuntu-20.04
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Install pipenv
|
|
run: |
|
|
pip3 install --upgrade pip setuptools wheel pipx
|
|
pipx install pipenv
|
|
-
|
|
name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: 3.9
|
|
cache: "pipenv"
|
|
cache-dependency-path: 'Pipfile.lock'
|
|
-
|
|
name: Install Python dependencies
|
|
run: |
|
|
pipenv sync --dev
|
|
-
|
|
name: Install system dependencies
|
|
run: |
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -qq --no-install-recommends gettext liblept5
|
|
-
|
|
name: Download frontend artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: frontend-compiled
|
|
path: src/documents/static/frontend/
|
|
-
|
|
name: Download documentation artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: documentation
|
|
path: docs/_build/html/
|
|
-
|
|
name: Generate requirements file
|
|
run: |
|
|
pipenv requirements > requirements.txt
|
|
-
|
|
name: Compile messages
|
|
run: |
|
|
cd src/
|
|
pipenv run python3 manage.py compilemessages
|
|
-
|
|
name: Collect static files
|
|
run: |
|
|
cd src/
|
|
pipenv run python3 manage.py collectstatic --no-input
|
|
-
|
|
name: Move files
|
|
run: |
|
|
mkdir dist
|
|
mkdir dist/paperless-ngx
|
|
mkdir dist/paperless-ngx/scripts
|
|
cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/
|
|
cp paperless.conf.example dist/paperless-ngx/paperless.conf
|
|
cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
|
|
cp -r docker/ dist/paperless-ngx/docker
|
|
cp scripts/*.service scripts/*.sh dist/paperless-ngx/scripts/
|
|
cp -r src/ dist/paperless-ngx/src
|
|
cp -r docs/_build/html/ dist/paperless-ngx/docs
|
|
mv static dist/paperless-ngx
|
|
-
|
|
name: Make release package
|
|
run: |
|
|
cd dist
|
|
tar -cJf paperless-ngx.tar.xz paperless-ngx/
|
|
-
|
|
name: Upload release artifact
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: release
|
|
path: dist/paperless-ngx.tar.xz
|
|
|
|
publish-release:
|
|
runs-on: ubuntu-20.04
|
|
outputs:
|
|
prerelease: ${{ steps.get_version.outputs.prerelease }}
|
|
changelog: ${{ steps.create-release.outputs.body }}
|
|
version: ${{ steps.get_version.outputs.version }}
|
|
needs:
|
|
- build-release
|
|
if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
|
|
steps:
|
|
-
|
|
name: Download release artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: release
|
|
path: ./
|
|
-
|
|
name: Get version
|
|
id: get_version
|
|
run: |
|
|
echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT
|
|
if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
|
|
echo "prerelease=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "prerelease=false" >> $GITHUB_OUTPUT
|
|
fi
|
|
-
|
|
name: Create Release and Changelog
|
|
id: create-release
|
|
uses: paperless-ngx/release-drafter@master
|
|
with:
|
|
name: Paperless-ngx ${{ steps.get_version.outputs.version }}
|
|
tag: ${{ steps.get_version.outputs.version }}
|
|
version: ${{ steps.get_version.outputs.version }}
|
|
prerelease: ${{ steps.get_version.outputs.prerelease }}
|
|
publish: true # ensures release is not marked as draft
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
-
|
|
name: Upload release archive
|
|
id: upload-release-asset
|
|
uses: shogo82148/actions-upload-release-asset@v1
|
|
with:
|
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
upload_url: ${{ steps.create-release.outputs.upload_url }}
|
|
asset_path: ./paperless-ngx.tar.xz
|
|
asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
|
|
asset_content_type: application/x-xz
|
|
|
|
append-changelog:
|
|
runs-on: ubuntu-20.04
|
|
needs:
|
|
- publish-release
|
|
if: needs.publish-release.outputs.prerelease == 'false'
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
with:
|
|
ref: main
|
|
-
|
|
name: Install pipenv
|
|
run: |
|
|
pip3 install --upgrade pip setuptools wheel pipx
|
|
pipx install pipenv
|
|
-
|
|
name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: 3.9
|
|
cache: "pipenv"
|
|
cache-dependency-path: 'Pipfile.lock'
|
|
-
|
|
name: Append Changelog to docs
|
|
id: append-Changelog
|
|
working-directory: docs
|
|
run: |
|
|
git branch ${{ needs.publish-release.outputs.version }}-changelog
|
|
git checkout ${{ needs.publish-release.outputs.version }}-changelog
|
|
echo -e "# Changelog\n\n${{ needs.publish-release.outputs.changelog }}\n" > changelog-new.md
|
|
echo "Manually linking usernames"
|
|
sed -i -r 's|@(.+?) \(\[#|[@\1](https://github.com/\1) ([#|ig' changelog-new.md
|
|
CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
|
|
echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
|
|
mv changelog-new.md changelog.md
|
|
pipenv run pre-commit run --files changelog.md
|
|
git config --global user.name "github-actions"
|
|
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA"
|
|
git push origin ${{ needs.publish-release.outputs.version }}-changelog
|
|
-
|
|
name: Create Pull Request
|
|
uses: actions/github-script@v6
|
|
with:
|
|
script: |
|
|
const { repo, owner } = context.repo;
|
|
const result = await github.rest.pulls.create({
|
|
title: '[Documentation] Add ${{ needs.publish-release.outputs.version }} changelog',
|
|
owner,
|
|
repo,
|
|
head: '${{ needs.publish-release.outputs.version }}-changelog',
|
|
base: 'main',
|
|
body: 'This PR is auto-generated by CI.'
|
|
});
|
|
github.rest.issues.addLabels({
|
|
owner,
|
|
repo,
|
|
issue_number: result.data.number,
|
|
labels: ['documentation']
|
|
});
|