paperless-ngx/.github/workflows/ci.yml

name: ci

on:
  push:
    tags:
      - ngx-*
      - beta-*
    branches-ignore:
      - 'translations**'
  pull_request:
    # We do not need synchronize event, it is covered by on:push
    types: [opened, reopened]
    branches-ignore:
      - 'translations**'

jobs:
  documentation:
    name: "Build Documentation"
    runs-on: ubuntu-20.04
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Install pipenv
        run: pipx install pipenv
      -
        name: Set up Python
        uses: actions/setup-python@v3
        with:
          python-version: 3.9
          cache: "pipenv"
          cache-dependency-path: 'Pipfile.lock'
      -
        name: Install dependencies
        run: |
          pipenv sync --dev
      -
        name: Make documentation
        run: |
          cd docs/
          pipenv run make html
      -
        name: Upload artifact
        uses: actions/upload-artifact@v3
        with:
          name: documentation
          path: docs/_build/html/

  code-checks-backend:
    name: "Backend Code Checks"
    runs-on: ubuntu-20.04
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Install checkers
        run: |
          pipx install reorder-python-imports
          pipx install yesqa
          pipx install add-trailing-comma
          pipx install flake8
      -
        name: Run reorder-python-imports
        run: |
          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
      -
        name: Run yesqa
        run: |
          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
      -
        name: Run add-trailing-comma
        run: |
          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
      # black is placed after add-trailing-comma because it may format differently
      # if a trailing comma is added
      -
        name: Run black
        uses: psf/black@stable
        with:
          options: "--check --diff"
          version: "22.3.0"
      -
        name: Run flake8 checks
        run: |
          cd src/
          flake8 --max-line-length=88 --ignore=E203,W503

  code-checks-frontend:
    name: "Frontend Code Checks"
    runs-on: ubuntu-20.04
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: '16'
      -
        name: Install prettier
        run: |
          npm install prettier
      -
        name: Run prettier
        run:
          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md

  tests-backend:
    needs: [code-checks-backend]
    name: "Backend Tests (${{ matrix.python-version }})"
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        python-version: ['3.8', '3.9']
      fail-fast: false
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 2
      -
        name: Install pipenv
        run: pipx install pipenv
      -
        name: Set up Python
        uses: actions/setup-python@v3
        with:
          python-version: "${{ matrix.python-version }}"
          cache: "pipenv"
          cache-dependency-path: 'Pipfile.lock'
      -
        name: Install system dependencies
        run: |
          sudo apt-get update -qq
          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng
      -
        name: Install Python dependencies
        run: |
          pipenv sync --dev
      -
        name: Tests
        run: |
          cd src/
          pipenv run pytest
      -
        name: Get changed files
        id: changed-files-specific
        uses: tj-actions/changed-files@v18.1
        with:
          files: |
            src/**
      -
        name: List all changed files
        run: |
          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
            echo "${file} was changed"
          done
      -
        name: Publish coverage results
        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        # https://github.com/coveralls-clients/coveralls-python/issues/251
        run: |
          cd src/
          pipenv run coveralls --service=github

  tests-frontend:
    needs: [code-checks-frontend]
    name: "Frontend Tests"
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        node-version: [16.x]
    steps:
      - uses: actions/checkout@v3
      - name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node-version }}
      - run: cd src-ui && npm ci
      - run: cd src-ui && npm run test
      - run: cd src-ui && npm run e2e:ci

  # build and push image to docker hub.
  build-docker-image:
    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
    runs-on: ubuntu-20.04
    needs: [tests-backend, tests-frontend]
    steps:
      -
        name: Gather Docker metadata
        id: docker-meta
        uses: docker/metadata-action@v3
        with:
          images: ghcr.io/${{ github.repository }}/paperless-ngx
          tags: |
            type=match,pattern=ngx-(\d.\d.\d),group=1
            type=semver,pattern=ngx-{{version}}
            type=semver,pattern=ngx-{{major}}.{{minor}}
            type=ref
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      -
        name: Login to Github Container Registry
        uses: docker/login-action@v1
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      -
        name: Build and push
        uses: docker/build-push-action@v2
        with:
          context: .
          file: ./Dockerfile
          platforms: linux/amd64,linux/arm/v7,linux/arm64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.docker-meta.outputs.tags }}
          labels: ${{ steps.docker-meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
      -
        name: Inspect image
        run: |
          docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
      -
        name: Export frontend artifact from docker
        run: |
          docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
          docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
      -
        name: Upload frontend artifact
        uses: actions/upload-artifact@v3
        with:
          name: frontend-compiled
          path: src/documents/static/frontend/

  build-release:
    needs: [build-docker-image, documentation]
    runs-on: ubuntu-20.04
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Set up Python
        uses: actions/setup-python@v3
        with:
          python-version: 3.9
      -
        name: Install dependencies
        run: |
          sudo apt-get update -qq
          sudo apt-get install -qq --no-install-recommends gettext liblept5
          pip3 install --upgrade pip setuptools wheel
          pip3 install -r requirements.txt
      -
        name: Download frontend artifact
        uses: actions/download-artifact@v3
        with:
          name: frontend-compiled
          path: src/documents/static/frontend/
      -
        name: Download documentation artifact
        uses: actions/download-artifact@v3
        with:
          name: documentation
          path: docs/_build/html/
      -
        name: Move files
        run: |
          mkdir dist
          mkdir dist/paperless-ngx
          mkdir dist/paperless-ngx/scripts
          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ngx/
          cp paperless.conf.example dist/paperless-ngx/paperless.conf
          cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
          cp docker/ dist/paperless-ngx/docker -r
          cp scripts/*.service scripts/*.sh dist/paperless-ngx/scripts/
          cp src/ dist/paperless-ngx/src -r
          cp docs/_build/html/ dist/paperless-ngx/docs -r
      -
        name: Compile messages
        run: |
          cd dist/paperless-ngx/src
          python3 manage.py compilemessages
      -
        name: Collect static files
        run: |
          cd dist/paperless-ngx/src
          python3 manage.py collectstatic --no-input
      -
        name: Make release package
        run: |
          cd dist
          find . -name __pycache__ | xargs rm -r
          tar -cJf paperless-ngx.tar.xz paperless-ngx/
      -
        name: Upload release artifact
        uses: actions/upload-artifact@v3
        with:
          name: release
          path: dist/paperless-ngx.tar.xz

  publish-release:
    runs-on: ubuntu-20.04
    needs: build-release
    if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
    steps:
      -
        name: Download release artifact
        uses: actions/download-artifact@v3
        with:
          name: release
          path: ./
      -
        name: Get version
        id: get_version
        run: |
          if [[ $GITHUB_REF == refs/tags/ngx-* ]]; then
            echo ::set-output name=version::${GITHUB_REF#refs/tags/ngx-}
            echo ::set-output name=prerelease::false
            echo ::set-output name=body::"For a complete list of changes, see the changelog at https://paperless-ngx.readthedocs.io/en/latest/changelog.html"
          elif [[ $GITHUB_REF == refs/tags/beta-* ]]; then
            echo ::set-output name=version::${GITHUB_REF#refs/tags/beta-}
            echo ::set-output name=prerelease::true
            echo ::set-output name=body::"For a complete list of changes, see the changelog at https://github.com/paperless-ngx/paperless-ngx/blob/beta/docs/changelog.rst"
          fi
      -
        name: Create release
        id: create_release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ngx-${{ steps.get_version.outputs.version }}
          release_name: Paperless-ngx ${{ steps.get_version.outputs.version }}
          draft: false
          prerelease: ${{ steps.get_version.outputs.prerelease }}
          body: ${{ steps.get_version.outputs.body }}
      -
        name: Upload release archive
        id: upload-release-asset
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
          asset_path: ./paperless-ngx.tar.xz
          asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
          asset_content_type: application/x-xz