paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into beta

Browse Source
This commit is contained in:
Trenton H
2022-11-09 13:51:10 -08:00
parent 34a0111ff5 f3b3db30a2
commit ba1366f49a
168 changed files with 19419 additions and 23870 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/ISSUE_TEMPLATE/bug-report.yml vendored
View File

@@ -13,6 +13,7 @@ body:
- [The troubleshooting documentation](https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html).
- [The installation instructions](https://paperless-ngx.readthedocs.io/en/latest/setup.html#installation).
- [Existing issues and discussions](https://github.com/paperless-ngx/paperless-ngx/search?q=&type=issues).
- Disable any customer container initialization scripts, if using any
If you encounter issues while installing or configuring Paperless-ngx, please post in the ["Support" section of the discussions](https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=support).
- type: textarea

470
.github/scripts/cleanup-tags.py vendored
View File

@@ -8,6 +8,7 @@ from argparse import ArgumentParser
from typing import Dict
from typing import Final
from typing import List
from typing import Optional
from common import get_log_level
from github import ContainerPackage
@@ -26,7 +27,7 @@ class DockerManifest2:
def __init__(self, data: Dict) -> None:
self._data = data
# This is the sha256: digest string. Corresponds to Github API name
# This is the sha256: digest string. Corresponds to GitHub API name
# if the package is an untagged package
self.digest = self._data["digest"]
platform_data_os = self._data["platform"]["os"]
@@ -38,6 +39,275 @@ class DockerManifest2:
self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
class RegistryTagsCleaner:
"""
This is the base class for the image registry cleaning. Given a package
name, it will keep all images which are tagged and all untagged images
referred to by a manifest. This results in only images which have been untagged
and cannot be referenced except by their SHA in being removed. None of these
images should be referenced, so it is fine to delete them.
"""
def __init__(
self,
package_name: str,
repo_owner: str,
repo_name: str,
package_api: GithubContainerRegistryApi,
branch_api: Optional[GithubBranchApi],
):
self.actually_delete = False
self.package_api = package_api
self.branch_api = branch_api
self.package_name = package_name
self.repo_owner = repo_owner
self.repo_name = repo_name
self.tags_to_delete: List[str] = []
self.tags_to_keep: List[str] = []
# Get the information about all versions of the given package
# These are active, not deleted, the default returned from the API
self.all_package_versions = self.package_api.get_active_package_versions(
self.package_name,
)
# Get a mapping from a tag like "1.7.0" or "feature-xyz" to the ContainerPackage
# tagged with it. It makes certain lookups easy
self.all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
for pkg in self.all_package_versions:
for tag in pkg.tags:
self.all_pkgs_tags_to_version[tag] = pkg
logger.info(
f"Located {len(self.all_package_versions)} versions of package {self.package_name}",
)
self.decide_what_tags_to_keep()
def clean(self):
"""
This method will delete image versions, based on the selected tags to delete
"""
for tag_to_delete in self.tags_to_delete:
package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
if self.actually_delete:
logger.info(
f"Deleting {tag_to_delete} (id {package_version_info.id})",
)
self.package_api.delete_package_version(
package_version_info,
)
else:
logger.info(
f"Would delete {tag_to_delete} (id {package_version_info.id})",
)
else:
logger.info("No tags to delete")
def clean_untagged(self, is_manifest_image: bool):
"""
This method will delete untagged images, that is those which are not named. It
handles if the image tag is actually a manifest, which points to images that look otherwise
untagged.
"""
def _clean_untagged_manifest():
"""
Handles the deletion of untagged images, but where the package is a manifest, ie a multi
arch image, which means some "untagged" images need to exist still.
Ok, bear with me, these are annoying.
Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
These images are untagged, but pointed to, and so should not be removed (or every pull fails).
So for each image getting kept, parse the manifest to find the digest(s) it points to. Then
remove those from the list of untagged images. The final result is the untagged, not pointed to
version which should be safe to remove.
Example:
Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
each of which appears as untagged image, but isn't really.
So from the list of untagged packages, remove those digests. Once all tags which
are being kept are checked, the remaining untagged packages are actually untagged
with no referrals in a manifest to them.
"""
# Simplify the untagged data, mapping name (which is a digest) to the version
# At the moment, these are the images which APPEAR untagged.
untagged_versions = {}
for x in self.all_package_versions:
if x.untagged:
untagged_versions[x.name] = x
skips = 0
# Parse manifests to locate digests pointed to
for tag in sorted(self.tags_to_keep):
full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
logger.info(f"Checking manifest for {full_name}")
try:
proc = subprocess.run(
[
shutil.which("docker"),
"manifest",
"inspect",
full_name,
],
capture_output=True,
)
manifest_list = json.loads(proc.stdout)
for manifest_data in manifest_list["manifests"]:
manifest = DockerManifest2(manifest_data)
if manifest.digest in untagged_versions:
logger.info(
f"Skipping deletion of {manifest.digest},"
f" referred to by {full_name}"
f" for {manifest.platform}",
)
del untagged_versions[manifest.digest]
skips += 1
except Exception as err:
self.actually_delete = False
logger.exception(err)
return
logger.info(
f"Skipping deletion of {skips} packages referred to by a manifest",
)
# Delete the untagged and not pointed at packages
logger.info(f"Deleting untagged packages of {self.package_name}")
for to_delete_name in untagged_versions:
to_delete_version = untagged_versions[to_delete_name]
if self.actually_delete:
logger.info(
f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
)
self.package_api.delete_package_version(
to_delete_version,
)
else:
logger.info(
f"Would delete {to_delete_name} (id {to_delete_version.id})",
)
def _clean_untagged_non_manifest():
"""
If the package is not a multi-arch manifest, images without tags are safe to delete.
"""
for package in self.all_package_versions:
if package.untagged:
if self.actually_delete:
logger.info(
f"Deleting id {package.id} named {package.name}",
)
self.package_api.delete_package_version(
package,
)
else:
logger.info(
f"Would delete {package.name} (id {package.id})",
)
else:
logger.info(
f"Not deleting tag {package.tags[0]} of package {self.package_name}",
)
logger.info("Beginning untagged image cleaning")
if is_manifest_image:
_clean_untagged_manifest()
else:
_clean_untagged_non_manifest()
def decide_what_tags_to_keep(self):
"""
This method holds the logic to delete what tags to keep and there fore
what tags to delete.
By default, any image with at least 1 tag will be kept
"""
# By default, keep anything which is tagged
self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
class MainImageTagsCleaner(RegistryTagsCleaner):
def decide_what_tags_to_keep(self):
"""
Overrides the default logic for deciding what images to keep. Images tagged as "feature-"
will be removed, if the corresponding branch no longer exists.
"""
# Default to everything gets kept still
super().decide_what_tags_to_keep()
# Locate the feature branches
feature_branches = {}
for branch in self.branch_api.get_branches(
repo=self.repo_name,
):
if branch.name.startswith("feature-"):
logger.debug(f"Found feature branch {branch.name}")
feature_branches[branch.name] = branch
logger.info(f"Located {len(feature_branches)} feature branches")
if not len(feature_branches):
# Our work here is done, delete nothing
return
# Filter to packages which are tagged with feature-*
packages_tagged_feature: List[ContainerPackage] = []
for package in self.all_package_versions:
if package.tag_matches("feature-"):
packages_tagged_feature.append(package)
# Map tags like "feature-xyz" to a ContainerPackage
feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
for pkg in packages_tagged_feature:
for tag in pkg.tags:
feature_pkgs_tags_to_versions[tag] = pkg
logger.info(
f'Located {len(feature_pkgs_tags_to_versions)} versions of package {self.package_name} tagged "feature-"',
)
# All the feature tags minus all the feature branches leaves us feature tags
# with no corresponding branch
self.tags_to_delete = list(
set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
)
# All the tags minus the set of going to be deleted tags leaves us the
# tags which will be kept around
self.tags_to_keep = list(
set(self.all_pkgs_tags_to_version.keys()) - set(self.tags_to_delete),
)
logger.info(
f"Located {len(self.tags_to_delete)} versions of package {self.package_name} to delete",
)
class LibraryTagsCleaner(RegistryTagsCleaner):
"""
Exists for the off change that someday, the installer library images
will need their own logic
"""
pass
def _main():
parser = ArgumentParser(
description="Using the GitHub API locate and optionally delete container"
@@ -100,190 +370,32 @@ def _main():
# Note: Only relevant to the main application, but simpler to
# leave in for all packages
with GithubBranchApi(gh_token) as branch_api:
feature_branches = {}
for branch in branch_api.get_branches(
repo=repo,
):
if branch.name.startswith("feature-"):
logger.debug(f"Found feature branch {branch.name}")
feature_branches[branch.name] = branch
logger.info(f"Located {len(feature_branches)} feature branches")
with GithubContainerRegistryApi(gh_token, repo_owner) as container_api:
# Get the information about all versions of the given package
all_package_versions: List[
ContainerPackage
] = container_api.get_package_versions(args.package)
all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
for pkg in all_package_versions:
for tag in pkg.tags:
all_pkgs_tags_to_version[tag] = pkg
logger.info(
f"Located {len(all_package_versions)} versions of package {args.package}",
)
# Filter to packages which are tagged with feature-*
packages_tagged_feature: List[ContainerPackage] = []
for package in all_package_versions:
if package.tag_matches("feature-"):
packages_tagged_feature.append(package)
feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
for pkg in packages_tagged_feature:
for tag in pkg.tags:
feature_pkgs_tags_to_versions[tag] = pkg
logger.info(
f'Located {len(feature_pkgs_tags_to_versions)} versions of package {args.package} tagged "feature-"',
)
# All the feature tags minus all the feature branches leaves us feature tags
# with no corresponding branch
tags_to_delete = list(
set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
)
# All the tags minus the set of going to be deleted tags leaves us the
# tags which will be kept around
tags_to_keep = list(
set(all_pkgs_tags_to_version.keys()) - set(tags_to_delete),
)
logger.info(
f"Located {len(tags_to_delete)} versions of package {args.package} to delete",
)
# Delete certain package versions for which no branch existed
for tag_to_delete in tags_to_delete:
package_version_info = feature_pkgs_tags_to_versions[tag_to_delete]
if args.delete:
logger.info(
f"Deleting {tag_to_delete} (id {package_version_info.id})",
with GithubContainerRegistryApi(gh_token, repo_owner) as container_api:
if args.package in {"paperless-ngx", "paperless-ngx/builder/cache/app"}:
cleaner = MainImageTagsCleaner(
args.package,
repo_owner,
repo,
container_api,
branch_api,
)
container_api.delete_package_version(
package_version_info,
)
else:
logger.info(
f"Would delete {tag_to_delete} (id {package_version_info.id})",
cleaner = LibraryTagsCleaner(
args.package,
repo_owner,
repo,
container_api,
None,
)
# Deal with untagged package versions
if args.untagged:
# Set if actually doing a delete vs dry run
cleaner.actually_delete = args.delete
logger.info("Handling untagged image packages")
# Clean images with tags
cleaner.clean()
if not args.is_manifest:
# If the package is not a multi-arch manifest, images without tags are safe to delete.
# They are not referred to by anything. This will leave all with at least 1 tag
for package in all_package_versions:
if package.untagged:
if args.delete:
logger.info(
f"Deleting id {package.id} named {package.name}",
)
container_api.delete_package_version(
package,
)
else:
logger.info(
f"Would delete {package.name} (id {package.id})",
)
else:
logger.info(
f"Not deleting tag {package.tags[0]} of package {args.package}",
)
else:
"""
Ok, bear with me, these are annoying.
Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
These images are untagged, but pointed to, and so should not be removed (or every pull fails).
So for each image getting kept, parse the manifest to find the digest(s) it points to. Then
remove those from the list of untagged images. The final result is the untagged, not pointed to
version which should be safe to remove.
Example:
Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
each of which appears as untagged image, but isn't really.
So from the list of untagged packages, remove those digests. Once all tags which
are being kept are checked, the remaining untagged packages are actually untagged
with no referrals in a manifest to them.
"""
# Simplify the untagged data, mapping name (which is a digest) to the version
untagged_versions = {}
for x in all_package_versions:
if x.untagged:
untagged_versions[x.name] = x
skips = 0
# Extra security to not delete on an unexpected error
actually_delete = True
# Parse manifests to locate digests pointed to
for tag in sorted(tags_to_keep):
full_name = f"ghcr.io/{repo_owner}/{args.package}:{tag}"
logger.info(f"Checking manifest for {full_name}")
try:
proc = subprocess.run(
[
shutil.which("docker"),
"manifest",
"inspect",
full_name,
],
capture_output=True,
)
manifest_list = json.loads(proc.stdout)
for manifest_data in manifest_list["manifests"]:
manifest = DockerManifest2(manifest_data)
if manifest.digest in untagged_versions:
logger.debug(
f"Skipping deletion of {manifest.digest}, referred to by {full_name} for {manifest.platform}",
)
del untagged_versions[manifest.digest]
skips += 1
except Exception as err:
actually_delete = False
logger.exception(err)
logger.info(
f"Skipping deletion of {skips} packages referred to by a manifest",
)
# Step 3.3 - Delete the untagged and not pointed at packages
logger.info(f"Deleting untagged packages of {args.package}")
for to_delete_name in untagged_versions:
to_delete_version = untagged_versions[to_delete_name]
if args.delete and actually_delete:
logger.info(
f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
)
container_api.delete_package_version(
to_delete_version,
)
else:
logger.info(
f"Would delete {to_delete_name} (id {to_delete_version.id})",
)
else:
logger.info("Leaving untagged images untouched")
# Clean images which are untagged
cleaner.clean_untagged(args.is_manifest)
if __name__ == "__main__":

5
.github/scripts/common.py vendored
View File

@@ -29,6 +29,11 @@ def get_cache_image_tag(
def get_log_level(args) -> int:
"""
Returns a logging level, based
:param args:
:return:
"""
levels = {
"critical": logging.CRITICAL,
"error": logging.ERROR,

75
.github/scripts/github.py vendored
View File

@@ -15,7 +15,7 @@ from typing import Dict
from typing import List
from typing import Optional
import requests
import httpx
logger = logging.getLogger("github-api")
@@ -28,15 +28,15 @@ class _GithubApiBase:
def __init__(self, token: str) -> None:
self._token = token
self._session: Optional[requests.Session] = None
self._client: Optional[httpx.Client] = None
def __enter__(self) -> "_GithubApiBase":
"""
Sets up the required headers for auth and response
type from the API
"""
self._session = requests.Session()
self._session.headers.update(
self._client = httpx.Client()
self._client.headers.update(
{
"Accept": "application/vnd.github.v3+json",
"Authorization": f"token {self._token}",
@@ -49,14 +49,14 @@ class _GithubApiBase:
Ensures the authorization token is cleaned up no matter
the reason for the exit
"""
if "Accept" in self._session.headers:
del self._session.headers["Accept"]
if "Authorization" in self._session.headers:
del self._session.headers["Authorization"]
if "Accept" in self._client.headers:
del self._client.headers["Accept"]
if "Authorization" in self._client.headers:
del self._client.headers["Authorization"]
# Close the session as well
self._session.close()
self._session = None
self._client.close()
self._client = None
def _read_all_pages(self, endpoint):
"""
@@ -66,7 +66,7 @@ class _GithubApiBase:
internal_data = []
while True:
resp = self._session.get(endpoint)
resp = self._client.get(endpoint)
if resp.status_code == 200:
internal_data += resp.json()
if "next" in resp.links:
@@ -76,7 +76,7 @@ class _GithubApiBase:
break
else:
logger.warning(f"Request to {endpoint} return HTTP {resp.status_code}")
break
resp.raise_for_status()
return internal_data
@@ -120,6 +120,7 @@ class GithubBranchApi(_GithubApiBase):
Returns all current branches of the given repository owned by the given
owner or organization.
"""
# The environment GITHUB_REPOSITORY already contains the owner in the correct location
endpoint = self._ENDPOINT.format(REPO=repo)
internal_data = self._read_all_pages(endpoint)
return [GithubBranch(branch) for branch in internal_data]
@@ -189,8 +190,11 @@ class GithubContainerRegistryApi(_GithubApiBase):
self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
# https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
self._PACKAGE_VERSION_RESTORE_ENDPOINT = (
f"{self._PACKAGE_VERSION_DELETE_ENDPOINT}/restore"
)
def get_package_versions(
def get_active_package_versions(
self,
package_name: str,
) -> List[ContainerPackage]:
@@ -216,12 +220,55 @@ class GithubContainerRegistryApi(_GithubApiBase):
return pkgs
def get_deleted_package_versions(
self,
package_name: str,
) -> List[ContainerPackage]:
package_type: str = "container"
# Need to quote this for slashes in the name
package_name = urllib.parse.quote(package_name, safe="")
endpoint = (
self._PACKAGES_VERSIONS_ENDPOINT.format(
ORG=self._owner_or_org,
PACKAGE_TYPE=package_type,
PACKAGE_NAME=package_name,
)
+ "?state=deleted"
)
pkgs = []
for data in self._read_all_pages(endpoint):
pkgs.append(ContainerPackage(data))
return pkgs
def delete_package_version(self, package_data: ContainerPackage):
"""
Deletes the given package version from the GHCR
"""
resp = self._session.delete(package_data.url)
resp = self._client.delete(package_data.url)
if resp.status_code != 204:
logger.warning(
f"Request to delete {package_data.url} returned HTTP {resp.status_code}",
)
def restore_package_version(
self,
package_name: str,
package_data: ContainerPackage,
):
package_type: str = "container"
endpoint = self._PACKAGE_VERSION_RESTORE_ENDPOINT.format(
ORG=self._owner_or_org,
PACKAGE_TYPE=package_type,
PACKAGE_NAME=package_name,
PACKAGE_VERSION_ID=package_data.id,
)
resp = self._client.post(endpoint)
if resp.status_code != 204:
logger.warning(
f"Request to delete {endpoint} returned HTTP {resp.status_code}",
)

63
.github/workflows/ci.yml vendored
View File

@@ -44,8 +44,7 @@ jobs:
-
name: Install pipenv
run: |
pipx install pipenv==2022.8.5
pipenv --version
pipx install pipenv==2022.10.12
-
name: Set up Python
uses: actions/setup-python@v4
@@ -82,17 +81,32 @@ jobs:
matrix:
python-version: ['3.8', '3.9', '3.10']
fail-fast: false
services:
tika:
image: ghcr.io/paperless-ngx/tika:latest
ports:
- "9998:9998/tcp"
gotenberg:
image: docker.io/gotenberg/gotenberg:7.6
ports:
- "3000:3000/tcp"
env:
# Enable Tika end to end testing
TIKA_LIVE: 1
# Enable paperless_mail testing against real server
PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 2
fetch-depth: 0
-
name: Install pipenv
run: |
pipx install pipenv==2022.8.5
pipenv --version
pipx install pipenv==2022.10.12
-
name: Set up Python
uses: actions/setup-python@v4
@@ -117,11 +131,11 @@ jobs:
name: Tests
run: |
cd src/
pipenv run pytest
pipenv run pytest -rfEp
-
name: Get changed files
id: changed-files-specific
uses: tj-actions/changed-files@v29.0.2
uses: tj-actions/changed-files@v34
with:
files: |
src/**
@@ -180,7 +194,7 @@ jobs:
id: set-ghcr-repository
run: |
ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
echo ::set-output name=repository::${ghcr_name}
echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
-
name: Checkout
uses: actions/checkout@v3
@@ -197,7 +211,7 @@ jobs:
echo ${build_json}
echo ::set-output name=qpdf-json::${build_json}
echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup psycopg2 image
id: psycopg2-setup
@@ -206,7 +220,7 @@ jobs:
echo ${build_json}
echo ::set-output name=psycopg2-json::${build_json}
echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup pikepdf image
id: pikepdf-setup
@@ -215,7 +229,7 @@ jobs:
echo ${build_json}
echo ::set-output name=pikepdf-json::${build_json}
echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup jbig2enc image
id: jbig2enc-setup
@@ -224,7 +238,7 @@ jobs:
echo ${build_json}
echo ::set-output name=jbig2enc-json::${build_json}
echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
outputs:
@@ -259,10 +273,10 @@ jobs:
run: |
if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
echo "Enabling DockerHub image push"
echo ::set-output name=enable::"true"
echo "enable=true" >> $GITHUB_OUTPUT
else
echo "Not pushing to DockerHub"
echo ::set-output name=enable::"false"
echo "enable=false" >> $GITHUB_OUTPUT
fi
-
name: Gather Docker metadata
@@ -443,11 +457,11 @@ jobs:
name: Get version
id: get_version
run: |
echo ::set-output name=version::${{ github.ref_name }}
echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT
if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
echo ::set-output name=prerelease::true
echo "prerelease=true" >> $GITHUB_OUTPUT
else
echo ::set-output name=prerelease::false
echo "prerelease=false" >> $GITHUB_OUTPUT
fi
-
name: Create Release and Changelog
@@ -484,6 +498,18 @@ jobs:
uses: actions/checkout@v3
with:
ref: main
-
name: Install pipenv
run: |
pip3 install --upgrade pip setuptools wheel pipx
pipx install pipenv
-
name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.9
cache: "pipenv"
cache-dependency-path: 'Pipfile.lock'
-
name: Append Changelog to docs
id: append-Changelog
@@ -497,9 +523,10 @@ jobs:
CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
mv changelog-new.md changelog.md
pipenv run pre-commit --files changelog.md
git config --global user.name "github-actions"
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git commit -am "Changelog ${{ steps.get_version.outputs.version }} - GHA"
git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA"
git push origin ${{ needs.publish-release.outputs.version }}-changelog
-
name: Create Pull Request

99
.github/workflows/cleanup-tags.yml vendored
View File

@@ -19,10 +19,31 @@ on:
- ".github/scripts/github.py"
- ".github/scripts/common.py"
concurrency:
group: registry-tags-cleanup
cancel-in-progress: false
jobs:
cleanup:
name: Cleanup Image Tags
runs-on: ubuntu-20.04
cleanup-images:
name: Cleanup Image Tags for ${{ matrix.primary-name }}
runs-on: ubuntu-latest
strategy:
matrix:
include:
- primary-name: "paperless-ngx"
cache-name: "paperless-ngx/builder/cache/app"
- primary-name: "paperless-ngx/builder/qpdf"
cache-name: "paperless-ngx/builder/cache/qpdf"
- primary-name: "paperless-ngx/builder/pikepdf"
cache-name: "paperless-ngx/builder/cache/pikepdf"
- primary-name: "paperless-ngx/builder/jbig2enc"
cache-name: "paperless-ngx/builder/cache/jbig2enc"
- primary-name: "paperless-ngx/builder/psycopg2"
cache-name: "paperless-ngx/builder/cache/psycopg2"
env:
# Requires a personal access token with the OAuth scope delete:packages
TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}
@@ -32,77 +53,43 @@ jobs:
uses: actions/checkout@v3
-
name: Login to Github Container Registry
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Set up Python
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: "3.10"
-
name: Install requests
name: Install httpx
run: |
python -m pip install requests
# Clean up primary packages
-
name: Cleanup for package "paperless-ngx"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx"
-
name: Cleanup for package "qpdf"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/qpdf"
-
name: Cleanup for package "pikepdf"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/pikepdf"
-
name: Cleanup for package "jbig2enc"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/jbig2enc"
-
name: Cleanup for package "psycopg2"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/psycopg2"
python -m pip install httpx
#
# Clean up registry cache packages
# Clean up primary package
#
-
name: Cleanup for package "builder/cache/app"
name: Cleanup for package "${{ matrix.primary-name }}"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/app"
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --is-manifest --delete "${{ matrix.primary-name }}"
#
# Clean up registry cache package
#
-
name: Cleanup for package "builder/cache/qpdf"
name: Cleanup for package "${{ matrix.cache-name }}"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/qpdf"
-
name: Cleanup for package "builder/cache/psycopg2"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/psycopg2"
-
name: Cleanup for package "builder/cache/jbig2enc"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/jbig2enc"
-
name: Cleanup for package "builder/cache/pikepdf"
if: "${{ env.TOKEN != '' }}"
run: |
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/pikepdf"
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}"
#
# Verify tags which are left still pull
#
-
name: Check all tags still pull
run: |
ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
echo "Pulling all tags of ghcr.io/${ghcr_name}"
docker pull --quiet --all-tags ghcr.io/${ghcr_name}
ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }')
echo "Pulling all tags of ${ghcr_name}"
docker pull --quiet --all-tags ${ghcr_name}
docker image list

2
.github/workflows/codeql-analysis.yml vendored
View File

@@ -38,7 +38,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL

35
.github/workflows/installer-library.yml vendored
View File

@@ -41,7 +41,7 @@ jobs:
id: set-ghcr-repository
run: |
ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
echo ::set-output name=repository::${ghcr_name}
echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
-
name: Checkout
uses: actions/checkout@v3
@@ -50,6 +50,11 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: "3.9"
-
name: Install jq
run: |
sudo apt-get update
sudo apt-get install jq
-
name: Setup qpdf image
id: qpdf-setup
@@ -58,7 +63,7 @@ jobs:
echo ${build_json}
echo ::set-output name=qpdf-json::${build_json}
echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup psycopg2 image
id: psycopg2-setup
@@ -67,7 +72,7 @@ jobs:
echo ${build_json}
echo ::set-output name=psycopg2-json::${build_json}
echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup pikepdf image
id: pikepdf-setup
@@ -76,7 +81,7 @@ jobs:
echo ${build_json}
echo ::set-output name=pikepdf-json::${build_json}
echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup jbig2enc image
id: jbig2enc-setup
@@ -85,7 +90,19 @@ jobs:
echo ${build_json}
echo ::set-output name=jbig2enc-json::${build_json}
echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup other versions
id: cache-bust-setup
run: |
pillow_version=$(jq ".default.pillow.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
lxml_version=$(jq ".default.lxml.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
echo "Pillow is ${pillow_version}"
echo "lxml is ${lxml_version}"
echo "pillow-version=${pillow_version}" >> $GITHUB_OUTPUT
echo "lxml-version=${lxml_version}" >> $GITHUB_OUTPUT
outputs:
@@ -97,7 +114,11 @@ jobs:
psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json }}
pillow-version: ${{ steps.cache-bust-setup.outputs.pillow-version }}
lxml-version: ${{ steps.cache-bust-setup.outputs.lxml-version }}
build-qpdf-debs:
name: qpdf
@@ -145,3 +166,5 @@ jobs:
REPO=${{ needs.prepare-docker-build.outputs.ghcr-repository }}
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}

4
.github/workflows/project-actions.yml vendored
View File

@@ -28,7 +28,7 @@ jobs:
if: github.event_name == 'issues' && (github.event.action == 'opened' || github.event.action == 'reopened')
steps:
- name: Add issue to project and set status to ${{ env.todo }}
uses: leonsteinhaeuser/project-beta-automations@v1.3.0
uses: leonsteinhaeuser/project-beta-automations@v2.0.1
with:
gh_token: ${{ secrets.GH_TOKEN }}
organization: paperless-ngx
@@ -44,7 +44,7 @@ jobs:
if: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'reopened') && github.event.pull_request.user.login != 'dependabot'
steps:
- name: Add PR to project and set status to "Needs Review"
uses: leonsteinhaeuser/project-beta-automations@v1.3.0
uses: leonsteinhaeuser/project-beta-automations@v2.0.1
with:
gh_token: ${{ secrets.GH_TOKEN }}
organization: paperless-ngx