Merge pull request #695 from paperless-ngx/beta

[Beta] Paperless-ngx v1.7.0 Release Candidate 1

.github/dependabot.yml

@@ -8,7 +8,7 @@ updates:
     target-branch: "dev"
     # Look for `package.json` and `lock` files in the `root` directory
     directory: "/src-ui"
-    # Check the npm registry for updates every week
+    # Check the npm registry for updates every month
     schedule:
       interval: "monthly"
     # Add reviewers
@@ -23,6 +23,19 @@ updates:
     # Check for updates once a week
     schedule:
       interval: "weekly"
+    labels:
+      - "backend"
+      - "dependencies"
+
+  # Enable updates for Github Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every month
+      interval: "monthly"
+    labels:
+      - "ci-cd"
+      - "dependencies"
     # Add reviewers
     reviewers:
       - "paperless-ngx/backend"

.github/workflows/ci.yml

@@ -13,90 +13,100 @@ on:
 
 jobs:
   documentation:
+    name: "Build Documentation"
     runs-on: ubuntu-20.04
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+      -
+        name: Install pipenv
+        run: pipx install pipenv
       -
         name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: 3.9
-      -
-        name: Get pip cache dir
-        id: pip-cache
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-      -
-        name: Persistent Github pip cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip3.8}
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
       -
         name: Install dependencies
         run: |
-          pip install --upgrade pipenv
-          pipenv install --system --dev --ignore-pipfile
+          pipenv sync --dev
       -
         name: Make documentation
         run: |
           cd docs/
-          make html
+          pipenv run make html
       -
         name: Upload artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: documentation
           path: docs/_build/html/
 
-  codestyle:
+  code-checks-backend:
+    name: "Backend Code Checks"
     runs-on: ubuntu-20.04
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       -
-        name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      -
-        name: Get pip cache dir
-        id: pip-cache
+        name: Install checkers
         run: |
-          echo "::set-output name=dir::$(pip cache dir)"
+          pipx install reorder-python-imports
+          pipx install yesqa
+          pipx install add-trailing-comma
+          pipx install flake8
       -
-        name: Persistent Github pip cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip${{ matrix.python-version }}
-      -
-        name: Install dependencies
+        name: Run reorder-python-imports
         run: |
-          pip install --upgrade pipenv
-          pipenv install --system --dev --ignore-pipfile
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
       -
-        name: Codestyle
+        name: Run yesqa
         run: |
-          cd src/
-          pycodestyle --max-line-length=88 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E203
-  codeformatting:
-    runs-on: ubuntu-20.04
-    steps:
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
       -
-        name: Checkout
-        uses: actions/checkout@v2
+        name: Run add-trailing-comma
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
+      # black is placed after add-trailing-comma because it may format differently
+      # if a trailing comma is added
       -
         name: Run black
         uses: psf/black@stable
         with:
           options: "--check --diff"
           version: "22.3.0"
+      -
+        name: Run flake8 checks
+        run: |
+          cd src/
+          flake8 --max-line-length=88 --ignore=E203,W503
 
-  tests:
+  code-checks-frontend:
+    name: "Frontend Code Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      -
+        name: Install prettier
+        run: |
+          npm install prettier
+      -
+        name: Run prettier
+        run:
+          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+
+  tests-backend:
+    needs: [code-checks-backend]
+    name: "Backend Tests (${{ matrix.python-version }})"
     runs-on: ubuntu-20.04
     strategy:
       matrix:
@@ -105,73 +115,94 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      -
+        name: Install pipenv
+        run: pipx install pipenv
       -
         name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: "${{ matrix.python-version }}"
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
       -
-        name: Get pip cache dir
-        id: pip-cache
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-      -
-        name: Persistent Github pip cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip${{ matrix.python-version }}
-      -
-        name: Install dependencies
+        name: Install system dependencies
         run: |
           sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng
-          pip install --upgrade pipenv
-          pipenv install --system --dev --ignore-pipfile
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
+      -
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
       -
         name: Tests
         run: |
           cd src/
-          pytest
+          pipenv run pytest
+      -
+        name: Get changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v18.1
+        with:
+          files: |
+            src/**
+      -
+        name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+            echo "${file} was changed"
+          done
       -
         name: Publish coverage results
-        if: matrix.python-version == '3.9'
+        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         # https://github.com/coveralls-clients/coveralls-python/issues/251
         run: |
           cd src/
-          coveralls --service=github
+          pipenv run coveralls --service=github
+
+  tests-frontend:
+    needs: [code-checks-frontend]
+    name: "Frontend Tests"
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run test
+      - run: cd src-ui && npm run e2e:ci
 
   # build and push image to docker hub.
   build-docker-image:
     if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
-    runs-on: ubuntu-latest
-    needs: [tests, codeformatting, codestyle]
+    concurrency:
+      group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
+      cancel-in-progress: true
+    runs-on: ubuntu-20.04
+    needs: [tests-backend, tests-frontend]
     steps:
       -
-        name: Prepare
-        id: prepare
-        run: |
-          IMAGE_NAME=ghcr.io/${{ github.repository }}
-          if [[ $GITHUB_REF == refs/tags/ngx-* ]]; then
-            TAGS=${IMAGE_NAME}:${GITHUB_REF#refs/tags/ngx-},${IMAGE_NAME}:latest
-            INSPECT_TAG=${IMAGE_NAME}:latest
-          elif [[ $GITHUB_REF == refs/tags/beta-* ]]; then
-            TAGS=${IMAGE_NAME}:beta
-            INSPECT_TAG=${TAGS}
-          elif [[ $GITHUB_REF == refs/heads/* ]]; then
-            TAGS=${IMAGE_NAME}:${GITHUB_REF#refs/heads/}
-            INSPECT_TAG=${TAGS}
-          else
-            exit 1
-          fi
-          echo ::set-output name=tags::${TAGS}
-          echo ::set-output name=inspect_tag::${INSPECT_TAG}
+        name: Gather Docker metadata
+        id: docker-meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=tag
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
@@ -192,36 +223,37 @@ jobs:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm/v7,linux/arm64
-          push: true
-          tags: ${{ steps.prepare.outputs.tags }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.docker-meta.outputs.tags }}
+          labels: ${{ steps.docker-meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
       -
         name: Inspect image
         run: |
-          docker buildx imagetools inspect ${{ steps.prepare.outputs.inspect_tag }}
+          docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
       -
         name: Export frontend artifact from docker
         run: |
-          docker run -d --name frontend-extract ${{ steps.prepare.outputs.inspect_tag }}
+          docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
           docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
       -
         name: Upload frontend artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: frontend-compiled
           path: src/documents/static/frontend/
 
   build-release:
-    needs: [build-docker-image, documentation, tests, codeformatting, codestyle]
+    needs: [build-docker-image, documentation]
     runs-on: ubuntu-20.04
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       -
         name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: 3.9
       -
@@ -233,13 +265,13 @@ jobs:
           pip3 install -r requirements.txt
       -
         name: Download frontend artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: frontend-compiled
           path: src/documents/static/frontend/
       -
         name: Download documentation artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: documentation
           path: docs/_build/html/
@@ -274,19 +306,19 @@ jobs:
           tar -cJf paperless-ngx.tar.xz paperless-ngx/
       -
         name: Upload release artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: release
           path: dist/paperless-ngx.tar.xz
 
   publish-release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: build-release
     if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
     steps:
       -
         name: Download release artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: release
           path: ./
@@ -297,24 +329,22 @@ jobs:
           if [[ $GITHUB_REF == refs/tags/ngx-* ]]; then
             echo ::set-output name=version::${GITHUB_REF#refs/tags/ngx-}
             echo ::set-output name=prerelease::false
-            echo ::set-output name=body::"For a complete list of changes, see the changelog at https://paperless-ngx.readthedocs.io/en/latest/changelog.html"
           elif [[ $GITHUB_REF == refs/tags/beta-* ]]; then
             echo ::set-output name=version::${GITHUB_REF#refs/tags/beta-}
             echo ::set-output name=prerelease::true
-            echo ::set-output name=body::"For a complete list of changes, see the changelog at https://github.com/paperless-ngx/paperless-ngx/blob/beta/docs/changelog.rst"
           fi
       -
-        name: Create release
-        id: create_release
-        uses: actions/create-release@v1
+        name: Create Release and Changelog
+        id: create-release
+        uses: release-drafter/release-drafter@v5
+        with:
+          name: Paperless-ngx ${{ steps.get_version.outputs.version }}
+          tag: ngx-${{ steps.get_version.outputs.version }}
+          version: ${{ steps.get_version.outputs.version }}
+          prerelease: ${{ steps.get_version.outputs.prerelease }}
+          publish: true # ensures release is not marked as draft
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ngx-${{ steps.get_version.outputs.version }}
-          release_name: Paperless-ngx ${{ steps.get_version.outputs.version }}
-          draft: false
-          prerelease: ${{ steps.get_version.outputs.prerelease }}
-          body: ${{ steps.get_version.outputs.body }}
       -
         name: Upload release archive
         id: upload-release-asset
@@ -322,7 +352,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          upload_url: ${{ steps.create-release.outputs.upload_url }}
           asset_path: ./paperless-ngx.tar.xz
           asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
           asset_content_type: application/x-xz