Chore: Backend bulk updates (#4509)

2025-04-02 13:45:10 -05:00 · 2023-11-13 09:09:56 -08:00 · 2023-11-13 09:09:56 -08:00 · facb7226fe
commit facb7226fe
parent b671f54cb7
15 changed files with 1764 additions and 1596 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -16,7 +16,7 @@ on:
 env:
  # This is the version of pipenv all the steps will use
  # If changing this, change Dockerfile
-  DEFAULT_PIP_ENV_VERSION: "2023.9.8"
+  DEFAULT_PIP_ENV_VERSION: "2023.10.24"
  # This is the default version of Python to use in most steps which aren't specific
  DEFAULT_PYTHON_VERSION: "3.10"

@ -64,7 +64,7 @@ jobs:
      -
        name: Install pipenv
        run: |
-          pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
+          pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
      -
        name: Install dependencies
        run: |
@ -133,7 +133,7 @@ jobs:
      -
        name: Install pipenv
        run: |
-          pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
+          pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
      -
        name: Install system dependencies
        run: |
@ -453,7 +453,7 @@ jobs:
      -
        name: Install pipenv + tools
        run: |
-          pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
+          pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
      -
        name: Install Python dependencies
        run: |
@ -621,7 +621,7 @@ jobs:
      -
        name: Install pipenv + tools
        run: |
-          pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
+          pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
      -
        name: Append Changelog to docs
        id: append-Changelog
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -5,7 +5,7 @@
 repos:
  # General hooks
  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
    hooks:
      - id: check-docstring-first
      - id: check-json
@ -27,7 +27,7 @@ repos:
      - id: check-case-conflict
      - id: detect-private-key
  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: 'v3.0.3'
+    rev: 'v3.1.0'
    hooks:
      - id: prettier
        types_or:
@ -37,16 +37,16 @@ repos:
        exclude: "(^Pipfile\\.lock$)"
  # Python hooks
  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.0.291'
+    rev: 'v0.1.5'
    hooks:
      - id: ruff
  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 23.9.1
+    rev: 23.11.0
    hooks:
      - id: black
  # Dockerfile hooks
  - repo: https://github.com/AleksaC/hadolint-py
-    rev: v2.12.0.2
+    rev: v2.12.0.3
    hooks:
      - id: hadolint
  # Shell script hooks
--- a/28
+++ b/28
@ -29,7 +29,7 @@ COPY Pipfile* ./

 RUN set -eux \
  && echo "Installing pipenv" \
-    && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.9.8 \
+    && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.10.24 \
  && echo "Generating requirement.txt" \
    && pipenv requirements > requirements.txt

@ -52,7 +52,7 @@ ARG TARGETARCH

 # Can be workflow provided, defaults set for manual building
 ARG JBIG2ENC_VERSION=0.29
-ARG QPDF_VERSION=11.6.1
+ARG QPDF_VERSION=11.6.3
 ARG GS_VERSION=10.02.0

 #
@ -112,19 +112,31 @@ RUN set -eux \
    && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
    && echo "Installing pre-built updates" \
      && echo "Installing qpdf ${QPDF_VERSION}" \
-        && curl --fail --silent --show-error --output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
-        && curl --fail --silent --show-error --output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
        && dpkg --install ./libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
        && dpkg --install ./qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
      && echo "Installing Ghostscript ${GS_VERSION}" \
-        && curl --fail --silent --show-error --output libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
-        && curl --fail --silent --show-error --output ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
-        && curl --fail --silent --show-error --output libgs10-common_${GS_VERSION}.dfsg-2_all.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
+        && curl --fail --silent --show-error --location \
+          --output libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
        && dpkg --install ./libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
        && dpkg --install ./libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
        && dpkg --install ./ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
      && echo "Installing jbig2enc" \
-        && curl --fail --silent --show-error --output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/jbig2enc-${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
        && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
      && echo "Cleaning up image layer" \
        && rm --force --verbose *.deb \
--- a/59
+++ b/59
@ -7,53 +7,53 @@ name = "pypi"
 dateparser = "~=1.1"
 # WARNING: django does not use semver.
 #          Only patch versions are guaranteed to not introduce breaking changes.
-django = "~=4.2.5"
-django-cors-headers = "*"
+django = "~=4.2.7"
+django-auditlog = "*"
 django-celery-results = "*"
 django-compression-middleware = "*"
-django-guardian = "*"
+django-cors-headers = "*"
 django-extensions = "*"
 django-filter = "~=23.3"
+django-guardian = "*"
+django-multiselectfield = "*"
 djangorestframework = "~=3.14"
 djangorestframework-guardian = "*"
 drf-writable-nested = "*"
+bleach = "*"
+celery = "*"
+channels = "~=4.0"
+channels-redis = "*"
+concurrent-log-handler = "*"
 filelock = "*"
+flower = "*"
+gotenberg-client = "*"
 gunicorn = "*"
 imap-tools = "*"
+inotifyrecursive = "~=0.3"
 langdetect = "*"
+mysqlclient = "*"
+nltk = "*"
+ocrmypdf = "~=15.0"
 pathvalidate = "*"
-python-gnupg = "*"
-python-dotenv = "*"
-python-dateutil = "*"
-python-magic = "*"
-python-ipware = "*"
+pdf2image = "*"
 psycopg2 = "*"
+python-dateutil = "*"
+python-dotenv = "*"
+python-gnupg = "*"
+python-ipware = "*"
+python-magic = "*"
+pyzbar = "*"
 rapidfuzz = "*"
 redis = {extras = ["hiredis"], version = "*"}
 scikit-learn = "~=1.3"
-whitenoise = "~=6.5"
-watchdog = "~=3.0"
-whoosh="~=2.7"
-inotifyrecursive = "~=0.3"
-ocrmypdf = "~=14.0"
-tqdm = "*"
-tika-client = "*"
-channels = "~=4.0"
-channels-redis = "*"
-uvicorn = {extras = ["standard"], version = "*"}
-concurrent-log-handler = "*"
-pyzbar = "*"
-mysqlclient = "*"
-celery = {extras = ["redis"], version = "*"}
 setproctitle = "*"
-nltk = "*"
-pdf2image = "*"
-flower = "*"
-bleach = "*"
+tika-client = "*"
+tqdm = "*"
+uvicorn = {extras = ["standard"], version = "*"}
+watchdog = "~=3.0"
+whitenoise = "~=6.6"
+whoosh="~=2.7"
 zxing-cpp = {version = "*", platform_machine = "== 'x86_64'"}
-django-multiselectfield = "*"
-gotenberg-client = "*"
-django-auditlog = "*"

 [dev-packages]
 # Linting
@ -70,7 +70,6 @@ pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
 pytest-rerunfailures = "*"
-"pdfminer.six" = "*"
 imagehash = "*"
 daphne = "*"
 # Documentation
--- a/Pipfile.lock
+++ b/Pipfile.lock
--- a/src-ui/src/app/components/common/permissions-dialog/permissions-dialog.component.ts
+++ b/src-ui/src/app/components/common/permissions-dialog/permissions-dialog.component.ts
@ -58,7 +58,8 @@ export class PermissionsDialogComponent {
  }

  @Input()
-  message = $localize`Note that permissions set here will override any existing permissions`
+  message =
+    $localize`Note that permissions set here will override any existing permissions`

  cancelClicked() {
    this.activeModal.close()
--- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts
+++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts
@ -476,8 +476,8 @@ export class BulkEditorComponent
      this.downloadForm.get('downloadFileTypeOriginals').value
        ? 'both'
        : this.downloadForm.get('downloadFileTypeArchive').value
-        ? 'archive'
-        : 'originals'
+          ? 'archive'
+          : 'originals'
    this.documentService
      .bulkDownload(
        Array.from(this.list.selected),
--- a/src/documents/classifier.py
+++ b/src/documents/classifier.py
@ -248,7 +248,7 @@ class DocumentClassifier:
        data_vectorized = self.data_vectorizer.fit_transform(content_generator())

        # See the notes here:
-        # https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html  # noqa: E501
+        # https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
        # This attribute isn't needed to function and can be large
        self.data_vectorizer.stop_words_ = None

--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@ -36,13 +36,13 @@ from documents.utils import copy_file_with_basic_stats
 # TODO: isnt there a date parsing library for this?

 DATE_REGEX = re.compile(
-    r"(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|"  # noqa: E501
-    r"(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|"  # noqa: E501
-    r"(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[a-zA-Z]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|"  # noqa: E501
+    r"(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|"
+    r"(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|"
+    r"(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[a-zA-Z]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|"
    r"(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{1,2}, ([0-9]{4}))(\b|(?=([_-])))|"
    r"(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{4})(\b|(?=([_-])))|"
-    r"(\b|(?!=([_-])))([0-9]{1,2}[^ ]{2}[\. ]+[^ ]{3,9}[ \.\/-][0-9]{4})(\b|(?=([_-])))|"  # noqa: E501
-    r"(\b|(?!=([_-])))(\b[0-9]{1,2}[ \.\/-][a-zA-Z]{3}[ \.\/-][0-9]{4})(\b|(?=([_-])))",  # noqa: E501
+    r"(\b|(?!=([_-])))([0-9]{1,2}[^ ]{2}[\. ]+[^ ]{3,9}[ \.\/-][0-9]{4})(\b|(?=([_-])))|"
+    r"(\b|(?!=([_-])))(\b[0-9]{1,2}[ \.\/-][a-zA-Z]{3}[ \.\/-][0-9]{4})(\b|(?=([_-])))",
 )


--- a/src/documents/serialisers.py
+++ b/src/documents/serialisers.py
@ -1023,7 +1023,6 @@ class AcknowledgeTasksViewSerializer(serializers.Serializer):
    )

    def _validate_task_id_list(self, tasks, name="tasks"):
-        pass
        if not isinstance(tasks, list):
            raise serializers.ValidationError(f"{name} must be a list")
        if not all(isinstance(i, int) for i in tasks):
--- a/src/documents/views.py
+++ b/src/documents/views.py
@ -158,10 +158,10 @@ class IndexView(TemplateView):
        context["main_js"] = f"frontend/{self.get_frontend_language()}/main.js"
        context[
            "webmanifest"
-        ] = f"frontend/{self.get_frontend_language()}/manifest.webmanifest"  # noqa: E501
+        ] = f"frontend/{self.get_frontend_language()}/manifest.webmanifest"
        context[
            "apple_touch_icon"
-        ] = f"frontend/{self.get_frontend_language()}/apple-touch-icon.png"  # noqa: E501
+        ] = f"frontend/{self.get_frontend_language()}/apple-touch-icon.png"
        return context


--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@ -469,7 +469,7 @@ SECRET_KEY = os.getenv(

 AUTH_PASSWORD_VALIDATORS = [
    {
-        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",  # noqa: E501
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
    },
    {
        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
@ -776,7 +776,7 @@ CONSUMER_IGNORE_PATTERNS = list(
    json.loads(
        os.getenv(
            "PAPERLESS_CONSUMER_IGNORE_PATTERNS",
-            '[".DS_Store", ".DS_STORE", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]',  # noqa: E501
+            '[".DS_Store", ".DS_STORE", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]',
        ),
    ),
 )
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@ -338,7 +338,7 @@ class RasterisedDocumentParser(DocumentParser):
            if "Ghostscript PDF/A rendering" in str(e):
                self.log.warning(
                    "Ghostscript PDF/A rendering failed, consider setting "
-                    "PAPERLESS_OCR_USER_ARGS: '{\"continue_on_soft_render_error\": true}'",  # noqa: E501
+                    "PAPERLESS_OCR_USER_ARGS: '{\"continue_on_soft_render_error\": true}'",
                )

            raise ParseError(
--- a/src/paperless_tika/signals.py
+++ b/src/paperless_tika/signals.py
@ -10,12 +10,12 @@ def tika_consumer_declaration(sender, **kwargs):
        "weight": 10,
        "mime_types": {
            "application/msword": ".doc",
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",  # noqa: E501
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
            "application/vnd.ms-excel": ".xls",
-            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",  # noqa: E501
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
            "application/vnd.ms-powerpoint": ".ppt",
-            "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",  # noqa: E501
-            "application/vnd.openxmlformats-officedocument.presentationml.slideshow": ".ppsx",  # noqa: E501
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+            "application/vnd.openxmlformats-officedocument.presentationml.slideshow": ".ppsx",
            "application/vnd.oasis.opendocument.presentation": ".odp",
            "application/vnd.oasis.opendocument.spreadsheet": ".ods",
            "application/vnd.oasis.opendocument.text": ".odt",
--- a/src/setup.cfg
+++ b/src/setup.cfg
@ -1,30 +1,23 @@
-[flake8]
-extend-exclude = */migrations/*, */tests/*
-# E203 - https://www.flake8rules.com/rules/E203.html
-# W503 - https://www.flake8rules.com/rules/W503.html
-ignore = E203,W503
-max-line-length = 88
-
 [tool:pytest]
-DJANGO_SETTINGS_MODULE=paperless.settings
+DJANGO_SETTINGS_MODULE = paperless.settings
 addopts = --pythonwarnings=all --cov --cov-report=html --cov-report=xml --numprocesses auto --maxprocesses=16 --quiet --durations=50
 env =
-  PAPERLESS_DISABLE_DBHANDLER=true
+    PAPERLESS_DISABLE_DBHANDLER=true

 [coverage:run]
 source =
-  ./
+    ./
 omit =
-  */tests/*
-	manage.py
-  paperless/workers.py
-	paperless/wsgi.py
-	paperless/auth.py
+    */tests/*
+    manage.py
+    paperless/workers.py
+    paperless/wsgi.py
+    paperless/auth.py

 [coverage:report]
 exclude_also =
    if settings.AUDIT_LOG_ENABLED:
-		if AUDIT_LOG_ENABLED:
+    if AUDIT_LOG_ENABLED:

 [mypy]
 plugins = mypy_django_plugin.main, mypy_drf_plugin.main, numpy.typing.mypy_plugin