mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-04-09 09:58:20 -05:00
f875ae4abf
Removes the building of installers from the repo, they can now be built elsewhere, on demand, as their building is no longer tied to the Dockerfile
537 lines
19 KiB
YAML
537 lines
19 KiB
YAML
name: ci
|
|
|
|
on:
|
|
push:
|
|
tags:
|
|
# https://semver.org/#spec-item-2
|
|
- 'v[0-9]+.[0-9]+.[0-9]+'
|
|
# https://semver.org/#spec-item-9
|
|
- 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+'
|
|
branches-ignore:
|
|
- 'translations**'
|
|
pull_request:
|
|
branches-ignore:
|
|
- 'translations**'
|
|
|
|
env:
|
|
# This is the version of pipenv all the steps will use
|
|
# If changing this, change Dockerfile
|
|
DEFAULT_PIP_ENV_VERSION: "2023.4.20"
|
|
# This is the default version of Python to use in most steps
|
|
# If changing this, change Dockerfile
|
|
DEFAULT_PYTHON_VERSION: "3.9"
|
|
|
|
jobs:
|
|
pre-commit:
|
|
name: Linting Checks
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
-
|
|
name: Checkout repository
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Install python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
|
-
|
|
name: Check files
|
|
uses: pre-commit/action@v3.0.0
|
|
|
|
documentation:
|
|
name: "Build Documentation"
|
|
runs-on: ubuntu-22.04
|
|
needs:
|
|
- pre-commit
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Set up Python
|
|
id: setup-python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
|
cache: "pipenv"
|
|
cache-dependency-path: 'Pipfile.lock'
|
|
-
|
|
name: Install pipenv
|
|
run: |
|
|
pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
|
|
-
|
|
name: Install dependencies
|
|
run: |
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
|
|
-
|
|
name: List installed Python dependencies
|
|
run: |
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list
|
|
-
|
|
name: Make documentation
|
|
run: |
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs build --config-file ./mkdocs.yml
|
|
-
|
|
name: Upload artifact
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: documentation
|
|
path: site/
|
|
|
|
documentation-deploy:
|
|
name: "Deploy Documentation"
|
|
runs-on: ubuntu-22.04
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
needs:
|
|
- documentation
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Deploy docs
|
|
uses: mhausenblas/mkdocs-deploy-gh-pages@master
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
CUSTOM_DOMAIN: docs.paperless-ngx.com
|
|
CONFIG_FILE: mkdocs.yml
|
|
EXTRA_PACKAGES: build-base
|
|
|
|
tests-backend:
|
|
name: "Tests (${{ matrix.python-version }})"
|
|
runs-on: ubuntu-22.04
|
|
needs:
|
|
- pre-commit
|
|
strategy:
|
|
matrix:
|
|
python-version: ['3.8', '3.9', '3.10']
|
|
fail-fast: false
|
|
env:
|
|
# Enable Tika end to end testing
|
|
TIKA_LIVE: 1
|
|
# Enable paperless_mail testing against real server
|
|
PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
|
|
PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
|
|
PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
|
|
# Enable Gotenberg end to end testing
|
|
GOTENBERG_LIVE: 1
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Start containers
|
|
run: |
|
|
docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml pull --quiet
|
|
docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml up --detach
|
|
-
|
|
name: Set up Python
|
|
id: setup-python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "${{ matrix.python-version }}"
|
|
cache: "pipenv"
|
|
cache-dependency-path: 'Pipfile.lock'
|
|
-
|
|
name: Install pipenv
|
|
run: |
|
|
pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
|
|
-
|
|
name: Install system dependencies
|
|
run: |
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
|
|
-
|
|
name: Configure ImageMagick
|
|
run: |
|
|
sudo cp docker/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
|
|
-
|
|
name: Install Python dependencies
|
|
run: |
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} run python --version
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
|
|
-
|
|
name: List installed Python dependencies
|
|
run: |
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list
|
|
-
|
|
name: Tests
|
|
run: |
|
|
cd src/
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra
|
|
-
|
|
name: Upload coverage to Codecov
|
|
if: ${{ matrix.python-version == env.DEFAULT_PYTHON_VERSION }}
|
|
uses: codecov/codecov-action@v3
|
|
with:
|
|
# not required for public repos, but intermittently fails otherwise
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
# future expansion
|
|
flags: backend
|
|
-
|
|
name: Stop containers
|
|
if: always()
|
|
run: |
|
|
docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml logs
|
|
docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml down
|
|
|
|
tests-frontend:
|
|
name: "Tests Frontend"
|
|
runs-on: ubuntu-22.04
|
|
needs:
|
|
- pre-commit
|
|
strategy:
|
|
matrix:
|
|
node-version: [16.x]
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
-
|
|
name: Use Node.js ${{ matrix.node-version }}
|
|
uses: actions/setup-node@v3
|
|
with:
|
|
node-version: ${{ matrix.node-version }}
|
|
cache: 'npm'
|
|
cache-dependency-path: 'src-ui/package-lock.json'
|
|
- run: cd src-ui && npm ci
|
|
- run: cd src-ui && npm run lint
|
|
- run: cd src-ui && npm run test
|
|
- run: cd src-ui && npm run e2e:ci
|
|
|
|
build-docker-image:
|
|
name: Build Docker image for ${{ github.ref_name }}
|
|
runs-on: ubuntu-22.04
|
|
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
|
|
concurrency:
|
|
group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
|
|
cancel-in-progress: true
|
|
needs:
|
|
- tests-backend
|
|
- tests-frontend
|
|
steps:
|
|
-
|
|
name: Check pushing to Docker Hub
|
|
id: push-other-places
|
|
# Only push to Dockerhub from the main repo AND the ref is either:
|
|
# main
|
|
# dev
|
|
# beta
|
|
# a tag
|
|
# Otherwise forks would require a Docker Hub account and secrets setup
|
|
run: |
|
|
if [[ ${{ github.repository_owner }} == "paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
|
|
echo "Enabling DockerHub image push"
|
|
echo "enable=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "Not pushing to DockerHub"
|
|
echo "enable=false" >> $GITHUB_OUTPUT
|
|
fi
|
|
-
|
|
name: Set ghcr repository name
|
|
id: set-ghcr-repository
|
|
run: |
|
|
ghcr_name=$(echo "${{ github.repository }}" | awk '{ print tolower($0) }')
|
|
echo "Name is ${ghcr_name}"
|
|
echo "ghcr-repository=${ghcr_name}" >> $GITHUB_OUTPUT
|
|
-
|
|
name: Gather Docker metadata
|
|
id: docker-meta
|
|
uses: docker/metadata-action@v4
|
|
with:
|
|
images: |
|
|
ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}
|
|
name=paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }}
|
|
name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }}
|
|
tags: |
|
|
# Tag branches with branch name
|
|
type=ref,event=branch
|
|
# Process semver tags
|
|
# For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag
|
|
type=semver,pattern={{version}}
|
|
type=semver,pattern={{major}}.{{minor}}
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
# If https://github.com/docker/buildx/issues/1044 is resolved,
|
|
# the append input with a native arm64 arch could be used to
|
|
# significantly speed up building
|
|
-
|
|
name: Set up Docker Buildx (fork/failure)
|
|
uses: docker/setup-buildx-action@v2
|
|
-
|
|
name: Set up QEMU
|
|
uses: docker/setup-qemu-action@v2
|
|
-
|
|
name: Login to Github Container Registry
|
|
uses: docker/login-action@v2
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
-
|
|
name: Login to Docker Hub
|
|
uses: docker/login-action@v2
|
|
# Don't attempt to login is not pushing to Docker Hub
|
|
if: steps.push-other-places.outputs.enable == 'true'
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
-
|
|
name: Login to Quay.io
|
|
uses: docker/login-action@v2
|
|
# Don't attempt to login is not pushing to Quay.io
|
|
if: steps.push-other-places.outputs.enable == 'true'
|
|
with:
|
|
registry: quay.io
|
|
username: ${{ secrets.QUAY_USERNAME }}
|
|
password: ${{ secrets.QUAY_ROBOT_TOKEN }}
|
|
-
|
|
name: Build and push
|
|
uses: docker/build-push-action@v4
|
|
with:
|
|
context: .
|
|
file: ./Dockerfile
|
|
platforms: linux/amd64,linux/arm/v7,linux/arm64
|
|
push: ${{ github.event_name != 'pull_request' }}
|
|
tags: ${{ steps.docker-meta.outputs.tags }}
|
|
labels: ${{ steps.docker-meta.outputs.labels }}
|
|
# Get cache layers from this branch, then dev, then main
|
|
# This allows new branches to get at least some cache benefits, generally from dev
|
|
cache-from: |
|
|
type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
|
|
type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:dev
|
|
cache-to: |
|
|
type=registry,mode=max,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
|
|
-
|
|
name: Inspect image
|
|
run: |
|
|
docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
|
|
-
|
|
name: Export frontend artifact from docker
|
|
run: |
|
|
docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
|
|
docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
|
|
-
|
|
name: Upload frontend artifact
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: frontend-compiled
|
|
path: src/documents/static/frontend/
|
|
|
|
build-release:
|
|
needs:
|
|
- build-docker-image
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Set up Python
|
|
id: setup-python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
|
cache: "pipenv"
|
|
cache-dependency-path: 'Pipfile.lock'
|
|
-
|
|
name: Install pipenv + tools
|
|
run: |
|
|
pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
|
|
-
|
|
name: Install Python dependencies
|
|
run: |
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
|
|
-
|
|
name: Install system dependencies
|
|
run: |
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -qq --no-install-recommends gettext liblept5
|
|
-
|
|
name: Download frontend artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: frontend-compiled
|
|
path: src/documents/static/frontend/
|
|
-
|
|
name: Download documentation artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: documentation
|
|
path: docs/_build/html/
|
|
-
|
|
name: Generate requirements file
|
|
run: |
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} requirements > requirements.txt
|
|
-
|
|
name: Compile messages
|
|
run: |
|
|
cd src/
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py compilemessages
|
|
-
|
|
name: Collect static files
|
|
run: |
|
|
cd src/
|
|
pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py collectstatic --no-input
|
|
-
|
|
name: Move files
|
|
run: |
|
|
echo "Making dist folders"
|
|
for directory in dist \
|
|
dist/paperless-ngx \
|
|
dist/paperless-ngx/scripts;
|
|
do
|
|
mkdir --verbose --parents ${directory}
|
|
done
|
|
|
|
echo "Copying basic files"
|
|
for file_name in .dockerignore \
|
|
.env \
|
|
Dockerfile \
|
|
Pipfile \
|
|
Pipfile.lock \
|
|
requirements.txt \
|
|
LICENSE \
|
|
README.md \
|
|
paperless.conf.example \
|
|
gunicorn.conf.py
|
|
do
|
|
cp --verbose ${file_name} dist/paperless-ngx/
|
|
done
|
|
mv --verbose dist/paperless-ngx/paperless.conf.example dist/paperless-ngx/paperless.conf
|
|
|
|
echo "Copying Docker related files"
|
|
cp --recursive docker/ dist/paperless-ngx/docker
|
|
|
|
echo "Copying startup scripts"
|
|
cp --verbose scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
|
|
|
|
echo "Copying source files"
|
|
cp --recursive src/ dist/paperless-ngx/src
|
|
echo "Copying documentation"
|
|
cp --recursive docs/_build/html/ dist/paperless-ngx/docs
|
|
|
|
mv --verbose static dist/paperless-ngx
|
|
-
|
|
name: Make release package
|
|
run: |
|
|
echo "Creating release archive"
|
|
cd dist
|
|
sudo chown -R 1000:1000 paperless-ngx/
|
|
tar -cJf paperless-ngx.tar.xz paperless-ngx/
|
|
-
|
|
name: Upload release artifact
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: release
|
|
path: dist/paperless-ngx.tar.xz
|
|
|
|
publish-release:
|
|
runs-on: ubuntu-22.04
|
|
outputs:
|
|
prerelease: ${{ steps.get_version.outputs.prerelease }}
|
|
changelog: ${{ steps.create-release.outputs.body }}
|
|
version: ${{ steps.get_version.outputs.version }}
|
|
needs:
|
|
- build-release
|
|
if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
|
|
steps:
|
|
-
|
|
name: Download release artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: release
|
|
path: ./
|
|
-
|
|
name: Get version
|
|
id: get_version
|
|
run: |
|
|
echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT
|
|
if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
|
|
echo "prerelease=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "prerelease=false" >> $GITHUB_OUTPUT
|
|
fi
|
|
-
|
|
name: Create Release and Changelog
|
|
id: create-release
|
|
uses: release-drafter/release-drafter@v5
|
|
with:
|
|
name: Paperless-ngx ${{ steps.get_version.outputs.version }}
|
|
tag: ${{ steps.get_version.outputs.version }}
|
|
version: ${{ steps.get_version.outputs.version }}
|
|
prerelease: ${{ steps.get_version.outputs.prerelease }}
|
|
publish: true # ensures release is not marked as draft
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
-
|
|
name: Upload release archive
|
|
id: upload-release-asset
|
|
uses: shogo82148/actions-upload-release-asset@v1
|
|
with:
|
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
upload_url: ${{ steps.create-release.outputs.upload_url }}
|
|
asset_path: ./paperless-ngx.tar.xz
|
|
asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
|
|
asset_content_type: application/x-xz
|
|
|
|
append-changelog:
|
|
runs-on: ubuntu-22.04
|
|
needs:
|
|
- publish-release
|
|
if: needs.publish-release.outputs.prerelease == 'false'
|
|
steps:
|
|
-
|
|
name: Checkout
|
|
uses: actions/checkout@v3
|
|
with:
|
|
ref: main
|
|
-
|
|
name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
|
cache: "pipenv"
|
|
cache-dependency-path: 'Pipfile.lock'
|
|
-
|
|
name: Install pipenv + tools
|
|
run: |
|
|
pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
|
|
-
|
|
name: Append Changelog to docs
|
|
id: append-Changelog
|
|
working-directory: docs
|
|
run: |
|
|
git branch ${{ needs.publish-release.outputs.version }}-changelog
|
|
git checkout ${{ needs.publish-release.outputs.version }}-changelog
|
|
echo -e "# Changelog\n\n${{ needs.publish-release.outputs.changelog }}\n" > changelog-new.md
|
|
echo "Manually linking usernames"
|
|
sed -i -r 's|@(.+?) \(\[#|[@\1](https://github.com/\1) ([#|ig' changelog-new.md
|
|
CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
|
|
echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
|
|
mv changelog-new.md changelog.md
|
|
pipenv run pre-commit run --files changelog.md || true
|
|
git config --global user.name "github-actions"
|
|
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA"
|
|
git push origin ${{ needs.publish-release.outputs.version }}-changelog
|
|
-
|
|
name: Create Pull Request
|
|
uses: actions/github-script@v6
|
|
with:
|
|
script: |
|
|
const { repo, owner } = context.repo;
|
|
const result = await github.rest.pulls.create({
|
|
title: '[Documentation] Add ${{ needs.publish-release.outputs.version }} changelog',
|
|
owner,
|
|
repo,
|
|
head: '${{ needs.publish-release.outputs.version }}-changelog',
|
|
base: 'main',
|
|
body: 'This PR is auto-generated by CI.'
|
|
});
|
|
github.rest.issues.addLabels({
|
|
owner,
|
|
repo,
|
|
issue_number: result.data.number,
|
|
labels: ['documentation', 'skip-changelog']
|
|
});
|