From 33e597f5bb0d0e0e2c3bb24e086c19b8de92b3c3 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Fri, 6 Jan 2023 08:00:09 -0800 Subject: [PATCH 1/2] Resolve testing not running with the correct python version --- .github/workflows/ci.yml | 80 +++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0015fd49..c24dc61a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ on: branches-ignore: - 'translations**' +env: + DEFAULT_PIP_ENV_VERSION: "2022.11.30" + DEFAULT_PYTHON_VERSION: "3.9" + jobs: pre-commit: name: Linting Checks @@ -21,13 +25,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - - name: Install tools + name: Install python uses: actions/setup-python@v4 with: - python-version: "3.9" - + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - name: Check files uses: pre-commit/action@v3.0.0 @@ -41,29 +43,30 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Install pipenv - run: | - pipx install pipenv==2022.11.30 - name: Set up Python + id: setup-python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} cache: "pipenv" cache-dependency-path: 'Pipfile.lock' + - + name: Install pipenv + run: | + pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION} - name: Install dependencies run: | - pipenv sync --dev + pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev - name: List installed Python dependencies run: | - pipenv run pip list + pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list - name: Make documentation run: | - pipenv run mkdocs build --config-file ./mkdocs.yml + pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs build --config-file ./mkdocs.yml - name: Upload artifact uses: actions/upload-artifact@v3 @@ -121,17 +124,18 @@ jobs: run: | docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml pull --quiet docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml up --detach - - - name: Install pipenv - run: | - pipx install pipenv==2022.11.30 - name: Set up Python + id: setup-python uses: actions/setup-python@v4 with: python-version: "${{ matrix.python-version }}" cache: "pipenv" cache-dependency-path: 'Pipfile.lock' + - + name: Install pipenv + run: | + pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION} - name: Install system dependencies run: | @@ -140,16 +144,17 @@ jobs: - name: Install Python dependencies run: | - pipenv sync --dev + pipenv --python ${{ steps.setup-python.outputs.python-version }} run python --version + pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev - name: List installed Python dependencies run: | - pipenv run pip list + pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list - name: Tests run: | cd src/ - pipenv run pytest -ra + pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra - name: Get changed files id: changed-files-specific @@ -165,13 +170,13 @@ jobs: done - name: Publish coverage results - if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true' + if: matrix.python-version == ${{ env.DEFAULT_PYTHON_VERSION }} && steps.changed-files-specific.outputs.any_changed == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/coveralls-clients/coveralls-python/issues/251 run: | cd src/ - pipenv run coveralls --service=github + pipenv --python ${{ steps.setup-python.outputs.python-version }} run coveralls --service=github - name: Stop containers if: always() @@ -227,7 +232,7 @@ jobs: name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - name: Setup qpdf image id: qpdf-setup @@ -389,22 +394,22 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Install pipenv - run: | - pip3 install --upgrade pip setuptools wheel pipx - pipx install pipenv - name: Set up Python + id: setup-python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} cache: "pipenv" cache-dependency-path: 'Pipfile.lock' + - + name: Install pipenv + tools + run: | + pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel - name: Install Python dependencies run: | - pipenv sync --dev + pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev - name: Install system dependencies run: | @@ -425,17 +430,17 @@ jobs: - name: Generate requirements file run: | - pipenv requirements > requirements.txt + pipenv --python ${{ steps.setup-python.outputs.python-version }} requirements > requirements.txt - name: Compile messages run: | cd src/ - pipenv run python3 manage.py compilemessages + pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py compilemessages - name: Collect static files run: | cd src/ - pipenv run python3 manage.py collectstatic --no-input + pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py collectstatic --no-input - name: Move files run: | @@ -522,18 +527,17 @@ jobs: uses: actions/checkout@v3 with: ref: main - - - name: Install pipenv - run: | - pip3 install --upgrade pip setuptools wheel pipx - pipx install pipenv - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} cache: "pipenv" cache-dependency-path: 'Pipfile.lock' + - + name: Install pipenv + tools + run: | + pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel - name: Append Changelog to docs id: append-Changelog From c21775980fa8621674a657eb99ea43d745101484 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Fri, 6 Jan 2023 10:20:58 -0800 Subject: [PATCH 2/2] relock with Python 3.8.15 --- .github/workflows/ci.yml | 4 ++++ .python-version | 1 + Pipfile | 3 --- Pipfile.lock | 31 ++++++++++++++++++++++++++++--- src/documents/parsers.py | 8 ++++---- 5 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 .python-version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c24dc61a2..daaadb42b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,11 @@ on: - 'translations**' env: + # This is the version of pipenv all the steps will use + # If changing this, change Dockerfile DEFAULT_PIP_ENV_VERSION: "2022.11.30" + # This is the default version of Python to use in most steps + # If changing this, change Dockerfile DEFAULT_PYTHON_VERSION: "3.9" jobs: diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..84691ddea --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.8.15 diff --git a/Pipfile b/Pipfile index 4309f5a2b..9d2e86112 100644 --- a/Pipfile +++ b/Pipfile @@ -44,9 +44,6 @@ channels = "~=3.0" uvicorn = {extras = ["standard"], version = "*"} concurrent-log-handler = "*" "pdfminer.six" = "*" -"backports.zoneinfo" = {version = "*", markers = "python_version < '3.9'"} -"importlib-resources" = {version = "*", markers = "python_version < '3.9'"} -zipp = {version = "*", markers = "python_version < '3.9'"} pyzbar = "*" mysqlclient = "*" celery = {extras = ["redis"], version = "*"} diff --git a/Pipfile.lock b/Pipfile.lock index 5d48c18dd..8e487b7d4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "cda8a69041c65176017700e723e8ae57849e0a98a0d2f02323c0078064e91db9" + "sha256": "5d6da0ede3fc7dd05c9a1d836bf8786285b10b6134e763d06ee90d6e1ccb2be7" }, "pipfile-spec": 6, "requires": {}, @@ -99,6 +99,7 @@ "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac", "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2" ], + "index": "pypi", "markers": "python_version < '3.9'", "version": "==0.2.1" }, @@ -1650,7 +1651,7 @@ "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa", "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e" ], - "markers": "python_version >= '3.7'", + "markers": "python_version < '3.10'", "version": "==4.4.0" }, "tzdata": { @@ -1899,7 +1900,7 @@ "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa", "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766" ], - "markers": "python_version < '3.9'", + "markers": "python_version < '3.10'", "version": "==3.11.0" }, "zope.interface": { @@ -2160,6 +2161,14 @@ "index": "pypi", "version": "==4.3.1" }, + "importlib-metadata": { + "hashes": [ + "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad", + "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d" + ], + "markers": "python_version < '3.10'", + "version": "==6.0.0" + }, "iniconfig": { "hashes": [ "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", @@ -2736,6 +2745,14 @@ "markers": "python_full_version < '3.11.0a7'", "version": "==2.0.1" }, + "typing-extensions": { + "hashes": [ + "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa", + "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e" + ], + "markers": "python_version < '3.10'", + "version": "==4.4.0" + }, "urllib3": { "hashes": [ "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", @@ -2785,6 +2802,14 @@ ], "index": "pypi", "version": "==2.2.0" + }, + "zipp": { + "hashes": [ + "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa", + "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766" + ], + "markers": "python_version < '3.10'", + "version": "==3.11.0" } } } diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 240d60e7f..cd17a33b1 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -6,7 +6,7 @@ import re import shutil import subprocess import tempfile -from functools import cache +from functools import lru_cache from typing import Iterator from typing import Match from typing import Optional @@ -45,7 +45,7 @@ DATE_REGEX = re.compile( logger = logging.getLogger("paperless.parsing") -@cache +@lru_cache(maxsize=8) def is_mime_type_supported(mime_type: str) -> bool: """ Returns True if the mime type is supported, False otherwise @@ -53,7 +53,7 @@ def is_mime_type_supported(mime_type: str) -> bool: return get_parser_class_for_mime_type(mime_type) is not None -@cache +@lru_cache(maxsize=8) def get_default_file_extension(mime_type: str) -> str: """ Returns the default file extension for a mimetype, or @@ -73,7 +73,7 @@ def get_default_file_extension(mime_type: str) -> str: return "" -@cache +@lru_cache(maxsize=8) def is_file_ext_supported(ext: str) -> bool: """ Returns True if the file extension is supported, False otherwise