diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0015fd49..daaadb42b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,14 @@ on: branches-ignore: - 'translations**' +env: + # This is the version of pipenv all the steps will use + # If changing this, change Dockerfile + DEFAULT_PIP_ENV_VERSION: "2022.11.30" + # This is the default version of Python to use in most steps + # If changing this, change Dockerfile + DEFAULT_PYTHON_VERSION: "3.9" + jobs: pre-commit: name: Linting Checks @@ -21,13 +29,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - - name: Install tools + name: Install python uses: actions/setup-python@v4 with: - python-version: "3.9" - + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - name: Check files uses: pre-commit/action@v3.0.0 @@ -41,29 +47,30 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Install pipenv - run: | - pipx install pipenv==2022.11.30 - name: Set up Python + id: setup-python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} cache: "pipenv" cache-dependency-path: 'Pipfile.lock' + - + name: Install pipenv + run: | + pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION} - name: Install dependencies run: | - pipenv sync --dev + pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev - name: List installed Python dependencies run: | - pipenv run pip list + pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list - name: Make documentation run: | - pipenv run mkdocs build --config-file ./mkdocs.yml + pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs build --config-file ./mkdocs.yml - name: Upload artifact uses: actions/upload-artifact@v3 @@ -121,17 +128,18 @@ jobs: run: | docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml pull --quiet docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml up --detach - - - name: Install pipenv - run: | - pipx install pipenv==2022.11.30 - name: Set up Python + id: setup-python uses: actions/setup-python@v4 with: python-version: "${{ matrix.python-version }}" cache: "pipenv" cache-dependency-path: 'Pipfile.lock' + - + name: Install pipenv + run: | + pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION} - name: Install system dependencies run: | @@ -140,16 +148,17 @@ jobs: - name: Install Python dependencies run: | - pipenv sync --dev + pipenv --python ${{ steps.setup-python.outputs.python-version }} run python --version + pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev - name: List installed Python dependencies run: | - pipenv run pip list + pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list - name: Tests run: | cd src/ - pipenv run pytest -ra + pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra - name: Get changed files id: changed-files-specific @@ -165,13 +174,13 @@ jobs: done - name: Publish coverage results - if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true' + if: matrix.python-version == ${{ env.DEFAULT_PYTHON_VERSION }} && steps.changed-files-specific.outputs.any_changed == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/coveralls-clients/coveralls-python/issues/251 run: | cd src/ - pipenv run coveralls --service=github + pipenv --python ${{ steps.setup-python.outputs.python-version }} run coveralls --service=github - name: Stop containers if: always() @@ -227,7 +236,7 @@ jobs: name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - name: Setup qpdf image id: qpdf-setup @@ -389,22 +398,22 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Install pipenv - run: | - pip3 install --upgrade pip setuptools wheel pipx - pipx install pipenv - name: Set up Python + id: setup-python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} cache: "pipenv" cache-dependency-path: 'Pipfile.lock' + - + name: Install pipenv + tools + run: | + pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel - name: Install Python dependencies run: | - pipenv sync --dev + pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev - name: Install system dependencies run: | @@ -425,17 +434,17 @@ jobs: - name: Generate requirements file run: | - pipenv requirements > requirements.txt + pipenv --python ${{ steps.setup-python.outputs.python-version }} requirements > requirements.txt - name: Compile messages run: | cd src/ - pipenv run python3 manage.py compilemessages + pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py compilemessages - name: Collect static files run: | cd src/ - pipenv run python3 manage.py collectstatic --no-input + pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py collectstatic --no-input - name: Move files run: | @@ -522,18 +531,17 @@ jobs: uses: actions/checkout@v3 with: ref: main - - - name: Install pipenv - run: | - pip3 install --upgrade pip setuptools wheel pipx - pipx install pipenv - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} cache: "pipenv" cache-dependency-path: 'Pipfile.lock' + - + name: Install pipenv + tools + run: | + pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel - name: Append Changelog to docs id: append-Changelog diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..84691ddea --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.8.15 diff --git a/Pipfile b/Pipfile index 0d4fcebae..3dd4cde3b 100644 --- a/Pipfile +++ b/Pipfile @@ -44,9 +44,6 @@ channels = "~=3.0" uvicorn = {extras = ["standard"], version = "*"} concurrent-log-handler = "*" "pdfminer.six" = "*" -"backports.zoneinfo" = {version = "*", markers = "python_version < '3.9'"} -"importlib-resources" = {version = "*", markers = "python_version < '3.9'"} -zipp = {version = "*", markers = "python_version < '3.9'"} pyzbar = "*" mysqlclient = "*" celery = {extras = ["redis"], version = "*"} diff --git a/Pipfile.lock b/Pipfile.lock index 540030266..ae7e69f35 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "bcce08eba4237bdef51daee4d88f26e624104b1907ed7bd8f93ac5ab27c995fa" + "sha256": "5d6da0ede3fc7dd05c9a1d836bf8786285b10b6134e763d06ee90d6e1ccb2be7" }, "pipfile-spec": 6, "requires": {}, @@ -1916,7 +1916,7 @@ "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa", "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766" ], - "markers": "python_version < '3.9'", + "markers": "python_version < '3.10'", "version": "==3.11.0" }, "zope.interface": { @@ -2761,31 +2761,6 @@ "markers": "python_full_version < '3.11.0a7'", "version": "==2.0.1" }, - "tornado": { - "hashes": [ - "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca", - "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72", - "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23", - "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8", - "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b", - "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9", - "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13", - "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75", - "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac", - "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", - "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" - ], - "markers": "python_version >= '3.7'", - "version": "==6.2" - }, - "tox": { - "hashes": [ - "sha256:5d214ce3e480e3b2cce72dbc9e832296387e7311f76b70de4a649636d02e34c9", - "sha256:d6b9f9f77796fcb1260d46f12dd4d6ebcc16bb73e72f7a683421b365491a912e" - ], - "index": "pypi", - "version": "==4.1.1" - }, "typing-extensions": { "hashes": [ "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa", @@ -2849,7 +2824,7 @@ "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa", "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766" ], - "markers": "python_version < '3.9'", + "markers": "python_version < '3.10'", "version": "==3.11.0" } } diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 240d60e7f..cd17a33b1 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -6,7 +6,7 @@ import re import shutil import subprocess import tempfile -from functools import cache +from functools import lru_cache from typing import Iterator from typing import Match from typing import Optional @@ -45,7 +45,7 @@ DATE_REGEX = re.compile( logger = logging.getLogger("paperless.parsing") -@cache +@lru_cache(maxsize=8) def is_mime_type_supported(mime_type: str) -> bool: """ Returns True if the mime type is supported, False otherwise @@ -53,7 +53,7 @@ def is_mime_type_supported(mime_type: str) -> bool: return get_parser_class_for_mime_type(mime_type) is not None -@cache +@lru_cache(maxsize=8) def get_default_file_extension(mime_type: str) -> str: """ Returns the default file extension for a mimetype, or @@ -73,7 +73,7 @@ def get_default_file_extension(mime_type: str) -> str: return "" -@cache +@lru_cache(maxsize=8) def is_file_ext_supported(ext: str) -> bool: """ Returns True if the file extension is supported, False otherwise