paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-10-30 03:56:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Chore: Backend bulk updates (#4509)

Browse Source
This commit is contained in:
Trenton H
2023-11-13 09:09:56 -08:00
committed by GitHub
parent b671f54cb7
commit facb7226fe
15 changed files with 1764 additions and 1596 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/ci.yml vendored
View File

@@ -16,7 +16,7 @@ on:
env:
# This is the version of pipenv all the steps will use
# If changing this, change Dockerfile
DEFAULT_PIP_ENV_VERSION: "2023.9.8"
DEFAULT_PIP_ENV_VERSION: "2023.10.24"
# This is the default version of Python to use in most steps which aren't specific
DEFAULT_PYTHON_VERSION: "3.10"
@@ -64,7 +64,7 @@ jobs:
-
name: Install pipenv
run: |
pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
-
name: Install dependencies
run: |
@@ -133,7 +133,7 @@ jobs:
-
name: Install pipenv
run: |
pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
-
name: Install system dependencies
run: |
@@ -453,7 +453,7 @@ jobs:
-
name: Install pipenv + tools
run: |
pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
-
name: Install Python dependencies
run: |
@@ -621,7 +621,7 @@ jobs:
-
name: Install pipenv + tools
run: |
pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
-
name: Append Changelog to docs
id: append-Changelog

10
.pre-commit-config.yaml
View File

@@ -5,7 +5,7 @@
repos:
# General hooks
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v4.5.0
hooks:
- id: check-docstring-first
- id: check-json
@@ -27,7 +27,7 @@ repos:
- id: check-case-conflict
- id: detect-private-key
- repo: https://github.com/pre-commit/mirrors-prettier
rev: 'v3.0.3'
rev: 'v3.1.0'
hooks:
- id: prettier
types_or:
@@ -37,16 +37,16 @@ repos:
exclude: "(^Pipfile\\.lock$)"
# Python hooks
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: 'v0.0.291'
rev: 'v0.1.5'
hooks:
- id: ruff
- repo: https://github.com/psf/black-pre-commit-mirror
rev: 23.9.1
rev: 23.11.0
hooks:
- id: black
# Dockerfile hooks
- repo: https://github.com/AleksaC/hadolint-py
rev: v2.12.0.2
rev: v2.12.0.3
hooks:
- id: hadolint
# Shell script hooks

28
Dockerfile
View File

@@ -29,7 +29,7 @@ COPY Pipfile* ./
RUN set -eux \
&& echo "Installing pipenv" \
&& python3 -m pip install --no-cache-dir --upgrade pipenv==2023.9.8 \
&& python3 -m pip install --no-cache-dir --upgrade pipenv==2023.10.24 \
&& echo "Generating requirement.txt" \
&& pipenv requirements > requirements.txt
@@ -52,7 +52,7 @@ ARG TARGETARCH
# Can be workflow provided, defaults set for manual building
ARG JBIG2ENC_VERSION=0.29
ARG QPDF_VERSION=11.6.1
ARG QPDF_VERSION=11.6.3
ARG GS_VERSION=10.02.0
#
@@ -112,19 +112,31 @@ RUN set -eux \
&& apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
&& echo "Installing pre-built updates" \
&& echo "Installing qpdf ${QPDF_VERSION}" \
&& curl --fail --silent --show-error --output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --location \
--output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --location \
--output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& dpkg --install ./libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& dpkg --install ./qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& echo "Installing Ghostscript ${GS_VERSION}" \
&& curl --fail --silent --show-error --output libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --output ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --output libgs10-common_${GS_VERSION}.dfsg-2_all.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
&& curl --fail --silent --show-error --location \
--output libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --location \
--output ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --location \
--output libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
&& dpkg --install ./libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
&& dpkg --install ./libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
&& dpkg --install ./ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
&& echo "Installing jbig2enc" \
&& curl --fail --silent --show-error --output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb --location https://github.com/paperless-ngx/builder/releases/download/v0.0.0/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --location \
--output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/jbig2enc-${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& echo "Cleaning up image layer" \
&& rm --force --verbose *.deb \

59
Pipfile
View File

@@ -7,53 +7,53 @@ name = "pypi"
dateparser = "~=1.1"
# WARNING: django does not use semver.
# Only patch versions are guaranteed to not introduce breaking changes.
django = "~=4.2.5"
django-cors-headers = "*"
django = "~=4.2.7"
django-auditlog = "*"
django-celery-results = "*"
django-compression-middleware = "*"
django-guardian = "*"
django-cors-headers = "*"
django-extensions = "*"
django-filter = "~=23.3"
django-guardian = "*"
django-multiselectfield = "*"
djangorestframework = "~=3.14"
djangorestframework-guardian = "*"
drf-writable-nested = "*"
bleach = "*"
celery = "*"
channels = "~=4.0"
channels-redis = "*"
concurrent-log-handler = "*"
filelock = "*"
flower = "*"
gotenberg-client = "*"
gunicorn = "*"
imap-tools = "*"
inotifyrecursive = "~=0.3"
langdetect = "*"
mysqlclient = "*"
nltk = "*"
ocrmypdf = "~=15.0"
pathvalidate = "*"
python-gnupg = "*"
python-dotenv = "*"
python-dateutil = "*"
python-magic = "*"
python-ipware = "*"
pdf2image = "*"
psycopg2 = "*"
python-dateutil = "*"
python-dotenv = "*"
python-gnupg = "*"
python-ipware = "*"
python-magic = "*"
pyzbar = "*"
rapidfuzz = "*"
redis = {extras = ["hiredis"], version = "*"}
scikit-learn = "~=1.3"
whitenoise = "~=6.5"
watchdog = "~=3.0"
whoosh="~=2.7"
inotifyrecursive = "~=0.3"
ocrmypdf = "~=14.0"
tqdm = "*"
tika-client = "*"
channels = "~=4.0"
channels-redis = "*"
uvicorn = {extras = ["standard"], version = "*"}
concurrent-log-handler = "*"
pyzbar = "*"
mysqlclient = "*"
celery = {extras = ["redis"], version = "*"}
setproctitle = "*"
nltk = "*"
pdf2image = "*"
flower = "*"
bleach = "*"
tika-client = "*"
tqdm = "*"
uvicorn = {extras = ["standard"], version = "*"}
watchdog = "~=3.0"
whitenoise = "~=6.6"
whoosh="~=2.7"
zxing-cpp = {version = "*", platform_machine = "== 'x86_64'"}
django-multiselectfield = "*"
gotenberg-client = "*"
django-auditlog = "*"
[dev-packages]
# Linting
@@ -70,7 +70,6 @@ pytest-env = "*"
pytest-sugar = "*"
pytest-xdist = "*"
pytest-rerunfailures = "*"
"pdfminer.six" = "*"
imagehash = "*"
daphne = "*"
# Documentation

3190
Pipfile.lock generated
View File

File diff suppressed because it is too large Load Diff

3
src-ui/src/app/components/common/permissions-dialog/permissions-dialog.component.ts
View File

@@ -58,7 +58,8 @@ export class PermissionsDialogComponent {
}
@Input()
message = $localize`Note that permissions set here will override any existing permissions`
message =
$localize`Note that permissions set here will override any existing permissions`
cancelClicked() {
this.activeModal.close()

4
src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts
View File

@@ -476,8 +476,8 @@ export class BulkEditorComponent
this.downloadForm.get('downloadFileTypeOriginals').value
? 'both'
: this.downloadForm.get('downloadFileTypeArchive').value
? 'archive'
: 'originals'
? 'archive'
: 'originals'
this.documentService
.bulkDownload(
Array.from(this.list.selected),

2
src/documents/classifier.py
View File

@@ -248,7 +248,7 @@ class DocumentClassifier:
data_vectorized = self.data_vectorizer.fit_transform(content_generator())
# See the notes here:
# https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html # noqa: E501
# https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
# This attribute isn't needed to function and can be large
self.data_vectorizer.stop_words_ = None

10
src/documents/parsers.py
View File

@@ -36,13 +36,13 @@ from documents.utils import copy_file_with_basic_stats
# TODO: isnt there a date parsing library for this?
DATE_REGEX = re.compile(
r"(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|" # noqa: E501
r"(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|" # noqa: E501
r"(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[a-zA-Z]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|" # noqa: E501
r"(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|"
r"(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|"
r"(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[a-zA-Z]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|"
r"(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{1,2}, ([0-9]{4}))(\b|(?=([_-])))|"
r"(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{4})(\b|(?=([_-])))|"
r"(\b|(?!=([_-])))([0-9]{1,2}[^ ]{2}[\. ]+[^ ]{3,9}[ \.\/-][0-9]{4})(\b|(?=([_-])))|" # noqa: E501
r"(\b|(?!=([_-])))(\b[0-9]{1,2}[ \.\/-][a-zA-Z]{3}[ \.\/-][0-9]{4})(\b|(?=([_-])))", # noqa: E501
r"(\b|(?!=([_-])))([0-9]{1,2}[^ ]{2}[\. ]+[^ ]{3,9}[ \.\/-][0-9]{4})(\b|(?=([_-])))|"
r"(\b|(?!=([_-])))(\b[0-9]{1,2}[ \.\/-][a-zA-Z]{3}[ \.\/-][0-9]{4})(\b|(?=([_-])))",
)

1
src/documents/serialisers.py
View File

@@ -1023,7 +1023,6 @@ class AcknowledgeTasksViewSerializer(serializers.Serializer):
)
def _validate_task_id_list(self, tasks, name="tasks"):
pass
if not isinstance(tasks, list):
raise serializers.ValidationError(f"{name} must be a list")
if not all(isinstance(i, int) for i in tasks):

4
src/documents/views.py
View File

@@ -158,10 +158,10 @@ class IndexView(TemplateView):
context["main_js"] = f"frontend/{self.get_frontend_language()}/main.js"
context[
"webmanifest"
] = f"frontend/{self.get_frontend_language()}/manifest.webmanifest" # noqa: E501
] = f"frontend/{self.get_frontend_language()}/manifest.webmanifest"
context[
"apple_touch_icon"
] = f"frontend/{self.get_frontend_language()}/apple-touch-icon.png" # noqa: E501
] = f"frontend/{self.get_frontend_language()}/apple-touch-icon.png"
return context

4
src/paperless/settings.py
View File

@@ -469,7 +469,7 @@ SECRET_KEY = os.getenv(
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa: E501
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
@@ -776,7 +776,7 @@ CONSUMER_IGNORE_PATTERNS = list(
json.loads(
os.getenv(
"PAPERLESS_CONSUMER_IGNORE_PATTERNS",
'[".DS_Store", ".DS_STORE", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]', # noqa: E501
'[".DS_Store", ".DS_STORE", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]',
),
),
)

2
src/paperless_tesseract/parsers.py
View File

@@ -338,7 +338,7 @@ class RasterisedDocumentParser(DocumentParser):
if "Ghostscript PDF/A rendering" in str(e):
self.log.warning(
"Ghostscript PDF/A rendering failed, consider setting "
"PAPERLESS_OCR_USER_ARGS: '{\"continue_on_soft_render_error\": true}'", # noqa: E501
"PAPERLESS_OCR_USER_ARGS: '{\"continue_on_soft_render_error\": true}'",
)
raise ParseError(

8
src/paperless_tika/signals.py
View File

@@ -10,12 +10,12 @@ def tika_consumer_declaration(sender, **kwargs):
"weight": 10,
"mime_types": {
"application/msword": ".doc",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", # noqa: E501
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
"application/vnd.ms-excel": ".xls",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx", # noqa: E501
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
"application/vnd.ms-powerpoint": ".ppt",
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx", # noqa: E501
"application/vnd.openxmlformats-officedocument.presentationml.slideshow": ".ppsx", # noqa: E501
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow": ".ppsx",
"application/vnd.oasis.opendocument.presentation": ".odp",
"application/vnd.oasis.opendocument.spreadsheet": ".ods",
"application/vnd.oasis.opendocument.text": ".odt",

25
src/setup.cfg
View File

@@ -1,30 +1,23 @@
[flake8]
extend-exclude = */migrations/*, */tests/*
# E203 - https://www.flake8rules.com/rules/E203.html
# W503 - https://www.flake8rules.com/rules/W503.html
ignore = E203,W503
max-line-length = 88
[tool:pytest]
DJANGO_SETTINGS_MODULE=paperless.settings
DJANGO_SETTINGS_MODULE = paperless.settings
addopts = --pythonwarnings=all --cov --cov-report=html --cov-report=xml --numprocesses auto --maxprocesses=16 --quiet --durations=50
env =
PAPERLESS_DISABLE_DBHANDLER=true
PAPERLESS_DISABLE_DBHANDLER=true
[coverage:run]
source =
./
./
omit =
*/tests/*
manage.py
paperless/workers.py
paperless/wsgi.py
paperless/auth.py
*/tests/*
manage.py
paperless/workers.py
paperless/wsgi.py
paperless/auth.py
[coverage:report]
exclude_also =
if settings.AUDIT_LOG_ENABLED:
if AUDIT_LOG_ENABLED:
if AUDIT_LOG_ENABLED:
[mypy]
plugins = mypy_django_plugin.main, mypy_drf_plugin.main, numpy.typing.mypy_plugin