Merge pull request #1316 from paperless-ngx/feature-docker-pipenv

Feature: Remove requirements.txt and use pipenv everywhere

.github/workflows/ci.yml

@@ -243,18 +243,27 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v3
+      -
+        name: Install pipenv
+        run: |
+          pip3 install --upgrade pip setuptools wheel pipx
+          pipx install pipenv
       -
         name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
       -
-        name: Install dependencies
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: Install system dependencies
         run: |
           sudo apt-get update -qq
           sudo apt-get install -qq --no-install-recommends gettext liblept5
-          pip3 install --upgrade pip setuptools wheel
-          pip3 install -r requirements.txt
       -
         name: Download frontend artifact
         uses: actions/download-artifact@v3
@@ -267,34 +276,38 @@ jobs:
         with:
           name: documentation
           path: docs/_build/html/
+      -
+        name: Generate requirements file
+        run: |
+          pipenv requirements > requirements.txt
+      -
+        name: Compile messages
+        run: |
+          cd src/
+          pipenv run python3 manage.py compilemessages
+      -
+        name: Collect static files
+        run: |
+          cd src/
+          pipenv run python3 manage.py collectstatic --no-input
       -
         name: Move files
         run: |
           mkdir dist
           mkdir dist/paperless-ngx
           mkdir dist/paperless-ngx/scripts
-          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ngx/
+          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/
           cp paperless.conf.example dist/paperless-ngx/paperless.conf
           cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
-          cp docker/ dist/paperless-ngx/docker -r
+          cp -r docker/ dist/paperless-ngx/docker
           cp scripts/*.service scripts/*.sh dist/paperless-ngx/scripts/
-          cp src/ dist/paperless-ngx/src -r
+          cp -r src/ dist/paperless-ngx/src
-          cp docs/_build/html/ dist/paperless-ngx/docs -r
+          cp -r docs/_build/html/ dist/paperless-ngx/docs
-      -
+          mv static dist/paperless-ngx
-        name: Compile messages
-        run: |
-          cd dist/paperless-ngx/src
-          python3 manage.py compilemessages
-      -
-        name: Collect static files
-        run: |
-          cd dist/paperless-ngx/src
-          python3 manage.py collectstatic --no-input
       -
         name: Make release package
         run: |
           cd dist
-          find . -name __pycache__ | xargs rm -r
           tar -cJf paperless-ngx.tar.xz paperless-ngx/
       -
         name: Upload release artifact

CODEOWNERS

@@ -7,4 +7,3 @@
 /src/ @paperless-ngx/backend
 Pipfile* @paperless-ngx/backend
 *.py @paperless-ngx/backend
-requirements.txt @paperless-ngx/backend

Dockerfile

@@ -175,9 +175,11 @@ RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
     && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
     && python3 -m pip list
 
+WORKDIR /usr/src/paperless/src/
+
 # Python dependencies
 # Change pretty frequently
-COPY requirements.txt ../
+COPY Pipfile* ./
 
 # Packages needed only for building a few quick Python
 # dependencies
@@ -191,20 +193,30 @@ RUN set -eux \
     && apt-get update \
     && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
     && python3 -m pip install --no-cache-dir --upgrade wheel \
+  && echo "Installing pipenv" \
+    && python3 -m pip install --no-cache-dir --upgrade pipenv \
   && echo "Installing Python requirements" \
-    && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+    # pipenv tries to be too fancy and prints so much junk
+    && pipenv requirements > requirements.txt \
+    && python3 -m pip install --default-timeout=1000 --no-cache-dir --requirement requirements.txt \
+    && rm requirements.txt \
   && echo "Cleaning up image" \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
     && apt-get clean --yes \
+    # Remove pipenv and its unique packages
+    && python3 -m pip uninstall --yes \
+      pipenv \
+      distlib \
+      platformdirs \
+      virtualenv \
+      virtualenv-clone \
     && rm -rf /var/lib/apt/lists/* \
     && rm -rf /tmp/* \
     && rm -rf /var/tmp/* \
     && rm -rf /var/cache/apt/archives/* \
     && truncate -s 0 /var/log/*log
 
-WORKDIR /usr/src/paperless/src/
-
 # copy backend
 COPY ./src ./
 

requirements.txt

@@ -1,108 +0,0 @@
--i https://pypi.python.org/simple
---extra-index-url https://www.piwheels.org/simple
-aioredis==1.3.1
-anyio==3.6.1; python_full_version >= '3.6.2'
-arrow==1.2.2; python_version >= '3.6'
-asgiref==3.5.2; python_version >= '3.7'
-async-timeout==4.0.2; python_version >= '3.6'
-attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-autobahn==22.6.1; python_version >= '3.7'
-automat==20.2.0
-backports.zoneinfo==0.2.1; python_version < '3.9'
-blessed==1.19.1; python_version >= '2.7'
-certifi==2022.6.15; python_version >= '3.6'
-cffi==1.15.1
-channels==3.0.5
-channels-redis==3.4.1
-charset-normalizer==2.1.0; python_version >= '3.6'
-click==8.1.3; python_version >= '3.7'
-coloredlogs==15.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-concurrent-log-handler==0.9.20
-constantly==15.1.0
-cryptography==37.0.4; python_version >= '3.6'
-daphne==3.0.2; python_version >= '3.6'
-dateparser==1.1.1
-deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-deprecation==2.1.0
-django==4.0.6
-django-cors-headers==3.13.0
-django-extensions==3.2.0
-django-filter==22.1
-django-picklefield==3.1; python_version >= '3'
--e git+https://github.com/paperless-ngx/django-q.git@bf20d57f859a7d872d5979cd8879fac9c9df981c#egg=django-q
-djangorestframework==3.13.1
-filelock==3.7.1
-fuzzywuzzy[speedup]==0.18.0
-gunicorn==20.1.0
-h11==0.13.0; python_version >= '3.6'
-hiredis==2.0.0; python_version >= '3.6'
-httptools==0.4.0
-humanfriendly==10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-hyperlink==21.0.0
-idna==3.3; python_version >= '3.5'
-imap-tools==0.56.0
-img2pdf==0.4.4
-importlib-resources==5.8.0; python_version < '3.9'
-incremental==21.3.0
-inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-inotifyrecursive==0.3.5
-joblib==1.1.0; python_version >= '3.6'
-langdetect==1.0.9
-lxml==4.9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-msgpack==1.0.4
-numpy==1.23.1; python_version >= '3.8'
-ocrmypdf==13.6.1
-packaging==21.3; python_version >= '3.6'
-pathvalidate==2.5.0
-pdf2image==1.16.0
-pdfminer.six==20220524
-pikepdf==5.4.0
-pillow==9.2.0
-pluggy==1.0.0; python_version >= '3.6'
-portalocker==2.5.1; python_version >= '3'
-psycopg2==2.9.3
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycparser==2.21
-pyopenssl==22.0.0
-pyparsing==3.0.9; python_full_version >= '3.6.8'
-python-dateutil==2.8.2
-python-dotenv==0.20.0
-python-gnupg==0.4.9
-python-levenshtein==0.12.2
-python-magic==0.4.27
-pytz==2022.1
-pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-pyyaml==6.0
-pyzbar==0.1.9
-redis==4.3.4
-regex==2022.3.2; python_version >= '3.6'
-reportlab==3.6.11; python_version >= '3.7' and python_version < '4'
-requests==2.28.1; python_version >= '3.7' and python_version < '4'
-scikit-learn==1.1.1
-scipy==1.8.1; python_version < '3.11' and python_version >= '3.8'
-service-identity==21.1.0
-setuptools==63.2.0; python_version >= '3.7'
-six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-sniffio==1.2.0; python_version >= '3.5'
-sqlparse==0.4.2; python_version >= '3.5'
-threadpoolctl==3.1.0; python_version >= '3.6'
-tika==1.24
-tqdm==4.64.0
-twisted[tls]==22.4.0; python_full_version >= '3.6.7'
-txaio==22.2.1; python_version >= '3.6'
-typing-extensions==4.3.0; python_version >= '3.7'
-tzdata==2022.1; python_version >= '3.6'
-tzlocal==4.2; python_version >= '3.6'
-urllib3==1.26.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
-uvicorn[standard]==0.18.2
-uvloop==0.16.0
-watchdog==2.1.9
-watchfiles==0.16.0
-wcwidth==0.2.5
-websockets==10.3
-whitenoise==6.2.0
-whoosh==2.7.4
-wrapt==1.14.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-zipp==3.8.1; python_version < '3.9'
-zope.interface==5.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'