Chore: Backend bulk updates (#4509)

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ on:
 env:
   # This is the version of pipenv all the steps will use
   # If changing this, change Dockerfile
-  DEFAULT_PIP_ENV_VERSION: "2023.9.8"
+  DEFAULT_PIP_ENV_VERSION: "2023.10.24"
   # This is the default version of Python to use in most steps which aren't specific
   DEFAULT_PYTHON_VERSION: "3.10"
 
@@ -64,7 +64,7 @@ jobs:
       -
         name: Install pipenv
         run: |
-          pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
+          pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
       -
         name: Install dependencies
         run: |
@@ -133,7 +133,7 @@ jobs:
       -
         name: Install pipenv
         run: |
-          pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
+          pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
       -
         name: Install system dependencies
         run: |
@@ -453,7 +453,7 @@ jobs:
       -
         name: Install pipenv + tools
         run: |
-          pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
+          pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
       -
         name: Install Python dependencies
         run: |
@@ -621,7 +621,7 @@ jobs:
       -
         name: Install pipenv + tools
         run: |
-          pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
+          pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
       -
         name: Append Changelog to docs
         id: append-Changelog

.pre-commit-config.yaml

@@ -5,7 +5,7 @@
 repos:
   # General hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-docstring-first
       - id: check-json
@@ -27,7 +27,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: 'v3.0.3'
+    rev: 'v3.1.0'
     hooks:
       - id: prettier
         types_or:
@@ -37,16 +37,16 @@ repos:
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.0.291'
+    rev: 'v0.1.5'
     hooks:
       - id: ruff
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 23.9.1
+    rev: 23.11.0
     hooks:
       - id: black
   # Dockerfile hooks
   - repo: https://github.com/AleksaC/hadolint-py
-    rev: v2.12.0.2
+    rev: v2.12.0.3
     hooks:
       - id: hadolint
   # Shell script hooks

Dockerfile

@@ -29,7 +29,7 @@ COPY Pipfile* ./
 
 RUN set -eux \
   && echo "Installing pipenv" \
-    && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.9.8 \
+    && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.10.24 \
   && echo "Generating requirement.txt" \
     && pipenv requirements > requirements.txt
 
@@ -52,7 +52,7 @@ ARG TARGETARCH
 
 # Can be workflow provided, defaults set for manual building
 ARG JBIG2ENC_VERSION=0.29
-ARG QPDF_VERSION=11.6.1
+ARG QPDF_VERSION=11.6.3
 ARG GS_VERSION=10.02.0
 
 #
@@ -112,19 +112,31 @@ RUN set -eux \
     && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
     && echo "Installing pre-built updates" \
       && echo "Installing qpdf ${QPDF_VERSION}" \
-        && curl --fail --silent --show-error --output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
-        && curl --fail --silent --show-error --output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+          --output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
         && dpkg --install ./libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
         && dpkg --install ./qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
       && echo "Installing Ghostscript ${GS_VERSION}" \
-        && curl --fail --silent --show-error --output libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
-        && curl --fail --silent --show-error --output ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+          --output libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
-        && curl --fail --silent --show-error --output libgs10-common_${GS_VERSION}.dfsg-2_all.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
         && dpkg --install ./libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
         && dpkg --install ./libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
         && dpkg --install ./ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
       && echo "Installing jbig2enc" \
-        && curl --fail --silent --show-error --output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/jbig2enc-${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
         && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
       && echo "Cleaning up image layer" \
         && rm --force --verbose *.deb \

Pipfile

@@ -7,53 +7,53 @@ name = "pypi"
 dateparser = "~=1.1"
 # WARNING: django does not use semver.
 #          Only patch versions are guaranteed to not introduce breaking changes.
-django = "~=4.2.5"
+django = "~=4.2.7"
-django-cors-headers = "*"
+django-auditlog = "*"
 django-celery-results = "*"
 django-compression-middleware = "*"
-django-guardian = "*"
+django-cors-headers = "*"
 django-extensions = "*"
 django-filter = "~=23.3"
+django-guardian = "*"
+django-multiselectfield = "*"
 djangorestframework = "~=3.14"
 djangorestframework-guardian = "*"
 drf-writable-nested = "*"
+bleach = "*"
+celery = "*"
+channels = "~=4.0"
+channels-redis = "*"
+concurrent-log-handler = "*"
 filelock = "*"
+flower = "*"
+gotenberg-client = "*"
 gunicorn = "*"
 imap-tools = "*"
+inotifyrecursive = "~=0.3"
 langdetect = "*"
+mysqlclient = "*"
+nltk = "*"
+ocrmypdf = "~=15.0"
 pathvalidate = "*"
-python-gnupg = "*"
+pdf2image = "*"
-python-dotenv = "*"
-python-dateutil = "*"
-python-magic = "*"
-python-ipware = "*"
 psycopg2 = "*"
+python-dateutil = "*"
+python-dotenv = "*"
+python-gnupg = "*"
+python-ipware = "*"
+python-magic = "*"
+pyzbar = "*"
 rapidfuzz = "*"
 redis = {extras = ["hiredis"], version = "*"}
 scikit-learn = "~=1.3"
-whitenoise = "~=6.5"
-watchdog = "~=3.0"
-whoosh="~=2.7"
-inotifyrecursive = "~=0.3"
-ocrmypdf = "~=14.0"
-tqdm = "*"
-tika-client = "*"
-channels = "~=4.0"
-channels-redis = "*"
-uvicorn = {extras = ["standard"], version = "*"}
-concurrent-log-handler = "*"
-pyzbar = "*"
-mysqlclient = "*"
-celery = {extras = ["redis"], version = "*"}
 setproctitle = "*"
-nltk = "*"
+tika-client = "*"
-pdf2image = "*"
+tqdm = "*"
-flower = "*"
+uvicorn = {extras = ["standard"], version = "*"}
-bleach = "*"
+watchdog = "~=3.0"
+whitenoise = "~=6.6"
+whoosh="~=2.7"
 zxing-cpp = {version = "*", platform_machine = "== 'x86_64'"}
-django-multiselectfield = "*"
-gotenberg-client = "*"
-django-auditlog = "*"
 
 [dev-packages]
 # Linting
@@ -70,7 +70,6 @@ pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
 pytest-rerunfailures = "*"
-"pdfminer.six" = "*"
 imagehash = "*"
 daphne = "*"
 # Documentation

src-ui/src/app/components/common/permissions-dialog/permissions-dialog.component.ts

@@ -58,7 +58,8 @@ export class PermissionsDialogComponent {
   }
 
   @Input()
-  message = $localize`Note that permissions set here will override any existing permissions`
+  message =
+    $localize`Note that permissions set here will override any existing permissions`
 
   cancelClicked() {
     this.activeModal.close()

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts

@@ -476,8 +476,8 @@ export class BulkEditorComponent
       this.downloadForm.get('downloadFileTypeOriginals').value
         ? 'both'
         : this.downloadForm.get('downloadFileTypeArchive').value
-        ? 'archive'
+          ? 'archive'
-        : 'originals'
+          : 'originals'
     this.documentService
       .bulkDownload(
         Array.from(this.list.selected),

src/documents/classifier.py

@@ -248,7 +248,7 @@ class DocumentClassifier:
         data_vectorized = self.data_vectorizer.fit_transform(content_generator())
 
         # See the notes here:
-        # https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html  # noqa: E501
+        # https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
         # This attribute isn't needed to function and can be large
         self.data_vectorizer.stop_words_ = None
 

src/documents/parsers.py

@@ -36,13 +36,13 @@ from documents.utils import copy_file_with_basic_stats
 # TODO: isnt there a date parsing library for this?
 
 DATE_REGEX = re.compile(
-    r"(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|"  # noqa: E501
+    r"(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|"
-    r"(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|"  # noqa: E501
+    r"(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|"
-    r"(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[a-zA-Z]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|"  # noqa: E501
+    r"(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[a-zA-Z]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|"
     r"(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{1,2}, ([0-9]{4}))(\b|(?=([_-])))|"
     r"(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{4})(\b|(?=([_-])))|"
-    r"(\b|(?!=([_-])))([0-9]{1,2}[^ ]{2}[\. ]+[^ ]{3,9}[ \.\/-][0-9]{4})(\b|(?=([_-])))|"  # noqa: E501
+    r"(\b|(?!=([_-])))([0-9]{1,2}[^ ]{2}[\. ]+[^ ]{3,9}[ \.\/-][0-9]{4})(\b|(?=([_-])))|"
-    r"(\b|(?!=([_-])))(\b[0-9]{1,2}[ \.\/-][a-zA-Z]{3}[ \.\/-][0-9]{4})(\b|(?=([_-])))",  # noqa: E501
+    r"(\b|(?!=([_-])))(\b[0-9]{1,2}[ \.\/-][a-zA-Z]{3}[ \.\/-][0-9]{4})(\b|(?=([_-])))",
 )
 
 

src/documents/serialisers.py

@@ -1023,7 +1023,6 @@ class AcknowledgeTasksViewSerializer(serializers.Serializer):
     )
 
     def _validate_task_id_list(self, tasks, name="tasks"):
-        pass
         if not isinstance(tasks, list):
             raise serializers.ValidationError(f"{name} must be a list")
         if not all(isinstance(i, int) for i in tasks):

src/documents/views.py

@@ -158,10 +158,10 @@ class IndexView(TemplateView):
         context["main_js"] = f"frontend/{self.get_frontend_language()}/main.js"
         context[
             "webmanifest"
-        ] = f"frontend/{self.get_frontend_language()}/manifest.webmanifest"  # noqa: E501
+        ] = f"frontend/{self.get_frontend_language()}/manifest.webmanifest"
         context[
             "apple_touch_icon"
-        ] = f"frontend/{self.get_frontend_language()}/apple-touch-icon.png"  # noqa: E501
+        ] = f"frontend/{self.get_frontend_language()}/apple-touch-icon.png"
         return context
 
 

src/paperless/settings.py

@@ -469,7 +469,7 @@ SECRET_KEY = os.getenv(
 
 AUTH_PASSWORD_VALIDATORS = [
     {
-        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",  # noqa: E501
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
     },
     {
         "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
@@ -776,7 +776,7 @@ CONSUMER_IGNORE_PATTERNS = list(
     json.loads(
         os.getenv(
             "PAPERLESS_CONSUMER_IGNORE_PATTERNS",
-            '[".DS_Store", ".DS_STORE", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]',  # noqa: E501
+            '[".DS_Store", ".DS_STORE", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]',
         ),
     ),
 )

src/paperless_tesseract/parsers.py

@@ -338,7 +338,7 @@ class RasterisedDocumentParser(DocumentParser):
             if "Ghostscript PDF/A rendering" in str(e):
                 self.log.warning(
                     "Ghostscript PDF/A rendering failed, consider setting "
-                    "PAPERLESS_OCR_USER_ARGS: '{\"continue_on_soft_render_error\": true}'",  # noqa: E501
+                    "PAPERLESS_OCR_USER_ARGS: '{\"continue_on_soft_render_error\": true}'",
                 )
 
             raise ParseError(

src/paperless_tika/signals.py

@@ -10,12 +10,12 @@ def tika_consumer_declaration(sender, **kwargs):
         "weight": 10,
         "mime_types": {
             "application/msword": ".doc",
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",  # noqa: E501
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
             "application/vnd.ms-excel": ".xls",
-            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",  # noqa: E501
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
             "application/vnd.ms-powerpoint": ".ppt",
-            "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",  # noqa: E501
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
-            "application/vnd.openxmlformats-officedocument.presentationml.slideshow": ".ppsx",  # noqa: E501
+            "application/vnd.openxmlformats-officedocument.presentationml.slideshow": ".ppsx",
             "application/vnd.oasis.opendocument.presentation": ".odp",
             "application/vnd.oasis.opendocument.spreadsheet": ".ods",
             "application/vnd.oasis.opendocument.text": ".odt",

src/setup.cfg

@@ -1,30 +1,23 @@
-[flake8]
-extend-exclude = */migrations/*, */tests/*
-# E203 - https://www.flake8rules.com/rules/E203.html
-# W503 - https://www.flake8rules.com/rules/W503.html
-ignore = E203,W503
-max-line-length = 88
-
 [tool:pytest]
-DJANGO_SETTINGS_MODULE=paperless.settings
+DJANGO_SETTINGS_MODULE = paperless.settings
 addopts = --pythonwarnings=all --cov --cov-report=html --cov-report=xml --numprocesses auto --maxprocesses=16 --quiet --durations=50
 env =
-  PAPERLESS_DISABLE_DBHANDLER=true
+    PAPERLESS_DISABLE_DBHANDLER=true
 
 [coverage:run]
 source =
-  ./
+    ./
 omit =
-  */tests/*
+    */tests/*
-	manage.py
+    manage.py
-  paperless/workers.py
+    paperless/workers.py
-	paperless/wsgi.py
+    paperless/wsgi.py
-	paperless/auth.py
+    paperless/auth.py
 
 [coverage:report]
 exclude_also =
     if settings.AUDIT_LOG_ENABLED:
-		if AUDIT_LOG_ENABLED:
+    if AUDIT_LOG_ENABLED:
 
 [mypy]
 plugins = mypy_django_plugin.main, mypy_drf_plugin.main, numpy.typing.mypy_plugin