Merge pull request #902 from paperless-ngx/v1.7.1

v1.7.1
This commit is contained in:
Quinn Casey
2022-05-10 16:11:12 -07:00
committed by GitHub
132 changed files with 37466 additions and 16326 deletions

View File

@@ -30,7 +30,6 @@ replacers: # Changes "Feature: Update checker" to "Update checker"
replace: ''
change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
change-title-escapes: '\<*_&#@'
tag-prefix: "ngx-"
template: |
# Changelog

27
.github/scripts/common.py vendored Normal file
View File

@@ -0,0 +1,27 @@
#!/usr/bin/env python3
def get_image_tag(
repo_name: str,
pkg_name: str,
pkg_version: str,
) -> str:
"""
Returns a string representing the normal image for a given package
"""
return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
def get_cache_image_tag(
repo_name: str,
pkg_name: str,
pkg_version: str,
branch_name: str,
) -> str:
"""
Returns a string representing the expected image cache tag for a given package
Registry type caching is utilized for the builder images, to allow fast
rebuilds, generally almost instant for the same version
"""
return f"ghcr.io/{repo_name}/builder/cache/{pkg_name}:{pkg_version}"

102
.github/scripts/get-build-json.py vendored Executable file
View File

@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""
This is a helper script for the mutli-stage Docker image builder.
It provides a single point of configuration for package version control.
The output JSON object is used by the CI workflow to determine what versions
to build and pull into the final Docker image.
Python package information is obtained from the Pipfile.lock. As this is
kept updated by dependabot, it usually will need no further configuration.
The sole exception currently is pikepdf, which has a dependency on qpdf,
and is configured here to use the latest version of qpdf built by the workflow.
Other package version information is configured directly below, generally by
setting the version and Git information, if any.
"""
import argparse
import json
import os
from pathlib import Path
from typing import Final
from common import get_cache_image_tag
from common import get_image_tag
def _main():
parser = argparse.ArgumentParser(
description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
)
parser.add_argument(
"package",
help="The name of the package to generate JSON for",
)
PIPFILE_LOCK_PATH: Final[Path] = Path("Pipfile.lock")
BUILD_CONFIG_PATH: Final[Path] = Path(".build-config.json")
# Read the main config file
build_json: Final = json.loads(BUILD_CONFIG_PATH.read_text())
# Read Pipfile.lock file
pipfile_data: Final = json.loads(PIPFILE_LOCK_PATH.read_text())
args: Final = parser.parse_args()
# Read from environment variables set by GitHub Actions
repo_name: Final[str] = os.environ["GITHUB_REPOSITORY"]
branch_name: Final[str] = os.environ["GITHUB_REF_NAME"]
# Default output values
version = None
git_tag = None
extra_config = {}
if args.package in pipfile_data["default"]:
# Read the version from Pipfile.lock
pkg_data = pipfile_data["default"][args.package]
pkg_version = pkg_data["version"].split("==")[-1]
version = pkg_version
# Based on the package, generate the expected Git tag name
if args.package == "pikepdf":
git_tag = f"v{pkg_version}"
elif args.package == "psycopg2":
git_tag = pkg_version.replace(".", "_")
# Any extra/special values needed
if args.package == "pikepdf":
extra_config["qpdf_version"] = build_json["qpdf"]["version"]
elif args.package in build_json:
version = build_json[args.package]["version"]
if "git_tag" in build_json[args.package]:
git_tag = build_json[args.package]["git_tag"]
else:
raise NotImplementedError(args.package)
# The JSON object we'll output
output = {
"name": args.package,
"version": version,
"git_tag": git_tag,
"image_tag": get_image_tag(repo_name, args.package, version),
"cache_tag": get_cache_image_tag(
repo_name,
args.package,
version,
branch_name,
),
}
# Add anything special a package may need
output.update(extra_config)
# Output the JSON info to stdout
print(json.dumps(output))
if __name__ == "__main__":
_main()

View File

@@ -3,8 +3,10 @@ name: ci
on:
push:
tags:
- ngx-*
- beta-*
# https://semver.org/#spec-item-2
- 'v[0-9]+.[0-9]+.[0-9]+'
# https://semver.org/#spec-item-9
- 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+'
branches-ignore:
- 'translations**'
pull_request:
@@ -45,161 +47,164 @@ jobs:
name: documentation
path: docs/_build/html/
code-checks-backend:
name: "Backend Code Checks"
runs-on: ubuntu-20.04
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Install checkers
run: |
pipx install reorder-python-imports
pipx install yesqa
pipx install add-trailing-comma
pipx install flake8
-
name: Run reorder-python-imports
run: |
find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-
name: Run yesqa
run: |
find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-
name: Run add-trailing-comma
run: |
find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
# black is placed after add-trailing-comma because it may format differently
# if a trailing comma is added
-
name: Run black
uses: psf/black@stable
with:
options: "--check --diff"
version: "22.3.0"
-
name: Run flake8 checks
run: |
cd src/
flake8 --max-line-length=88 --ignore=E203,W503
ci-backend:
uses: ./.github/workflows/reusable-ci-backend.yml
code-checks-frontend:
name: "Frontend Code Checks"
runs-on: ubuntu-20.04
steps:
-
name: Checkout
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16'
-
name: Install prettier
run: |
npm install prettier
-
name: Run prettier
run:
npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
ci-frontend:
uses: ./.github/workflows/reusable-ci-frontend.yml
tests-backend:
needs: [code-checks-backend]
name: "Backend Tests (${{ matrix.python-version }})"
prepare-docker-build:
name: Prepare Docker Pipeline Data
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ['3.8', '3.9']
fail-fast: false
needs:
- documentation
- ci-backend
- ci-frontend
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 2
-
name: Install pipenv
run: pipx install pipenv
-
name: Set up Python
uses: actions/setup-python@v3
with:
python-version: "${{ matrix.python-version }}"
cache: "pipenv"
cache-dependency-path: 'Pipfile.lock'
python-version: "3.9"
-
name: Install system dependencies
name: Setup qpdf image
id: qpdf-setup
run: |
sudo apt-get update -qq
sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
-
name: Install Python dependencies
run: |
pipenv sync --dev
-
name: Tests
run: |
cd src/
pipenv run pytest
-
name: Get changed files
id: changed-files-specific
uses: tj-actions/changed-files@v18.1
with:
files: |
src/**
-
name: List all changed files
run: |
for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
echo "${file} was changed"
done
-
name: Publish coverage results
if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# https://github.com/coveralls-clients/coveralls-python/issues/251
run: |
cd src/
pipenv run coveralls --service=github
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
tests-frontend:
needs: [code-checks-frontend]
name: "Frontend Tests"
runs-on: ubuntu-20.04
strategy:
matrix:
node-version: [16.x]
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- run: cd src-ui && npm ci
- run: cd src-ui && npm run test
- run: cd src-ui && npm run e2e:ci
echo ${build_json}
echo ::set-output name=qpdf-json::${build_json}
-
name: Setup psycopg2 image
id: psycopg2-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
echo ${build_json}
echo ::set-output name=psycopg2-json::${build_json}
-
name: Setup pikepdf image
id: pikepdf-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
echo ${build_json}
echo ::set-output name=pikepdf-json::${build_json}
-
name: Setup jbig2enc image
id: jbig2enc-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
echo ${build_json}
echo ::set-output name=jbig2enc-json::${build_json}
outputs:
qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
build-qpdf-debs:
name: qpdf
needs:
- prepare-docker-build
uses: ./.github/workflows/reusable-workflow-builder.yml
with:
dockerfile: ./docker-builders/Dockerfile.qpdf
build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
build-args: |
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
build-jbig2enc:
name: jbig2enc
needs:
- prepare-docker-build
uses: ./.github/workflows/reusable-workflow-builder.yml
with:
dockerfile: ./docker-builders/Dockerfile.jbig2enc
build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
build-args: |
JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
build-psycopg2-wheel:
name: psycopg2
needs:
- prepare-docker-build
uses: ./.github/workflows/reusable-workflow-builder.yml
with:
dockerfile: ./docker-builders/Dockerfile.psycopg2
build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
build-args: |
PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
build-pikepdf-wheel:
name: pikepdf
needs:
- prepare-docker-build
- build-qpdf-debs
uses: ./.github/workflows/reusable-workflow-builder.yml
with:
dockerfile: ./docker-builders/Dockerfile.pikepdf
build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
build-args: |
REPO=${{ github.repository }}
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
# build and push image to docker hub.
build-docker-image:
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
concurrency:
group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
cancel-in-progress: true
runs-on: ubuntu-20.04
needs: [tests-backend, tests-frontend]
concurrency:
group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
cancel-in-progress: true
needs:
- prepare-docker-build
- build-psycopg2-wheel
- build-jbig2enc
- build-qpdf-debs
- build-pikepdf-wheel
steps:
-
name: Check pushing to Docker Hub
id: docker-hub
# Only push to Dockerhub from the main repo
# Otherwise forks would require a Docker Hub account and secrets setup
run: |
if [[ ${{ github.repository }} == "paperless-ngx/paperless-ngx" ]] ; then
echo ::set-output name=enable::"true"
else
echo ::set-output name=enable::"false"
fi
-
name: Gather Docker metadata
id: docker-meta
uses: docker/metadata-action@v3
with:
images: ghcr.io/${{ github.repository }}
images: |
ghcr.io/${{ github.repository }}
name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
tags: |
# Tag branches with branch name
type=ref,event=branch
type=ref,event=tag
# Process semver tags
# For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
-
name: Checkout
uses: actions/checkout@v3
@@ -216,6 +221,14 @@ jobs:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Login to Docker Hub
uses: docker/login-action@v1
# Don't attempt to login is not pushing to Docker Hub
if: steps.docker-hub.outputs.enable == 'true'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build and push
uses: docker/build-push-action@v2
@@ -226,8 +239,19 @@ jobs:
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
# Get cache layers from this branch, then dev, then main
# This allows new branches to get at least some cache benefits, generally from dev
cache-from: |
type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:dev
type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:main
cache-to: |
type=registry,mode=max,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
-
name: Inspect image
run: |
@@ -245,7 +269,8 @@ jobs:
path: src/documents/static/frontend/
build-release:
needs: [build-docker-image, documentation]
needs:
- build-docker-image
runs-on: ubuntu-20.04
steps:
-
@@ -313,8 +338,9 @@ jobs:
publish-release:
runs-on: ubuntu-20.04
needs: build-release
if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
needs:
- build-release
if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
steps:
-
name: Download release artifact
@@ -326,12 +352,11 @@ jobs:
name: Get version
id: get_version
run: |
if [[ $GITHUB_REF == refs/tags/ngx-* ]]; then
echo ::set-output name=version::${GITHUB_REF#refs/tags/ngx-}
echo ::set-output name=prerelease::false
elif [[ $GITHUB_REF == refs/tags/beta-* ]]; then
echo ::set-output name=version::${GITHUB_REF#refs/tags/beta-}
echo ::set-output name=version::${{ github.ref_name }}
if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
echo ::set-output name=prerelease::true
else
echo ::set-output name=prerelease::false
fi
-
name: Create Release and Changelog
@@ -339,7 +364,7 @@ jobs:
uses: release-drafter/release-drafter@v5
with:
name: Paperless-ngx ${{ steps.get_version.outputs.version }}
tag: ngx-${{ steps.get_version.outputs.version }}
tag: ${{ steps.get_version.outputs.version }}
version: ${{ steps.get_version.outputs.version }}
prerelease: ${{ steps.get_version.outputs.prerelease }}
publish: true # ensures release is not marked as draft

View File

@@ -42,7 +42,7 @@ jobs:
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,4 +51,4 @@ jobs:
# queries: ./path/to/local/query, your-org/your-repo/queries@main
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1
uses: github/codeql-action/analyze@v2

View File

@@ -0,0 +1,108 @@
name: Backend CI Jobs
on:
workflow_call:
jobs:
code-checks-backend:
name: "Code Style Checks"
runs-on: ubuntu-20.04
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Install checkers
run: |
pipx install reorder-python-imports
pipx install yesqa
pipx install add-trailing-comma
pipx install flake8
-
name: Run reorder-python-imports
run: |
find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-
name: Run yesqa
run: |
find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-
name: Run add-trailing-comma
run: |
find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
# black is placed after add-trailing-comma because it may format differently
# if a trailing comma is added
-
name: Run black
uses: psf/black@stable
with:
options: "--check --diff"
version: "22.3.0"
-
name: Run flake8 checks
run: |
cd src/
flake8 --max-line-length=88 --ignore=E203,W503
tests-backend:
name: "Tests (${{ matrix.python-version }})"
runs-on: ubuntu-20.04
needs:
- code-checks-backend
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10']
fail-fast: false
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 2
-
name: Install pipenv
run: pipx install pipenv
-
name: Set up Python
uses: actions/setup-python@v3
with:
python-version: "${{ matrix.python-version }}"
cache: "pipenv"
cache-dependency-path: 'Pipfile.lock'
-
name: Install system dependencies
run: |
sudo apt-get update -qq
sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
-
name: Install Python dependencies
run: |
pipenv sync --dev
-
name: Tests
run: |
cd src/
pipenv run pytest
-
name: Get changed files
id: changed-files-specific
uses: tj-actions/changed-files@v19
with:
files: |
src/**
-
name: List all changed files
run: |
for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
echo "${file} was changed"
done
-
name: Publish coverage results
if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# https://github.com/coveralls-clients/coveralls-python/issues/251
run: |
cd src/
pipenv run coveralls --service=github

View File

@@ -0,0 +1,42 @@
name: Frontend CI Jobs
on:
workflow_call:
jobs:
code-checks-frontend:
name: "Code Style Checks"
runs-on: ubuntu-20.04
steps:
-
name: Checkout
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16'
-
name: Install prettier
run: |
npm install prettier
-
name: Run prettier
run:
npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
tests-frontend:
name: "Tests"
runs-on: ubuntu-20.04
needs:
- code-checks-frontend
strategy:
matrix:
node-version: [16.x]
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- run: cd src-ui && npm ci
- run: cd src-ui && npm run test
- run: cd src-ui && npm run e2e:ci

View File

@@ -0,0 +1,53 @@
name: Reusable Image Builder
on:
workflow_call:
inputs:
dockerfile:
required: true
type: string
build-json:
required: true
type: string
build-args:
required: false
default: ""
type: string
concurrency:
group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
cancel-in-progress: false
jobs:
build-image:
name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Login to Github Container Registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Build ${{ fromJSON(inputs.build-json).name }}
uses: docker/build-push-action@v2
with:
context: .
file: ${{ inputs.dockerfile }}
tags: ${{ fromJSON(inputs.build-json).image_tag }}
platforms: linux/amd64,linux/arm64,linux/arm/v7
build-args: ${{ inputs.build-args }}
push: true
cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }}
cache-to: type=registry,mode=max,ref=${{ fromJSON(inputs.build-json).cache_tag }}