mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-04-09 09:58:20 -05:00
Simplifies the image tag cleanup into classes, fixes the library images caring about feature branches
This commit is contained in:
parent
69913ae250
commit
c3a62268c7
436
.github/scripts/cleanup-tags.py
vendored
436
.github/scripts/cleanup-tags.py
vendored
@ -1,4 +1,12 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
This script cleans up the untagged images of the main image. It checks for "feature-"
|
||||||
|
branches, correlates them to the images, and removes images which have no branch
|
||||||
|
related to them.
|
||||||
|
|
||||||
|
After removing the image, it looks at untagged images, removing those which are
|
||||||
|
not pointed to by a manifest.
|
||||||
|
"""
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
@ -8,6 +16,7 @@ from argparse import ArgumentParser
|
|||||||
from typing import Dict
|
from typing import Dict
|
||||||
from typing import Final
|
from typing import Final
|
||||||
from typing import List
|
from typing import List
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
from common import get_log_level
|
from common import get_log_level
|
||||||
from github import ContainerPackage
|
from github import ContainerPackage
|
||||||
@ -26,7 +35,7 @@ class DockerManifest2:
|
|||||||
|
|
||||||
def __init__(self, data: Dict) -> None:
|
def __init__(self, data: Dict) -> None:
|
||||||
self._data = data
|
self._data = data
|
||||||
# This is the sha256: digest string. Corresponds to Github API name
|
# This is the sha256: digest string. Corresponds to GitHub API name
|
||||||
# if the package is an untagged package
|
# if the package is an untagged package
|
||||||
self.digest = self._data["digest"]
|
self.digest = self._data["digest"]
|
||||||
platform_data_os = self._data["platform"]["os"]
|
platform_data_os = self._data["platform"]["os"]
|
||||||
@ -38,6 +47,236 @@ class DockerManifest2:
|
|||||||
self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
|
self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
|
||||||
|
|
||||||
|
|
||||||
|
class RegistryTagsCleaner:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
package_name: str,
|
||||||
|
repo_owner: str,
|
||||||
|
repo_name: str,
|
||||||
|
package_api: GithubContainerRegistryApi,
|
||||||
|
branch_api: Optional[GithubBranchApi],
|
||||||
|
):
|
||||||
|
self.actually_delete = False
|
||||||
|
self.package_api = package_api
|
||||||
|
self.branch_api = branch_api
|
||||||
|
self.package_name = package_name
|
||||||
|
self.repo_owner = repo_owner
|
||||||
|
self.repo_name = repo_name
|
||||||
|
self.tags_to_delete: List[str] = []
|
||||||
|
self.tags_to_keep: List[str] = []
|
||||||
|
|
||||||
|
# Get the information about all versions of the given package
|
||||||
|
# These are active, not deleted, the default returned from the API
|
||||||
|
self.all_package_versions = self.package_api.get_active_package_versions(
|
||||||
|
self.package_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Get a mapping from a tag like "1.7.0" or "feature-xyz" to the ContainerPackage
|
||||||
|
# tagged with it. It makes certain lookups easy
|
||||||
|
self.all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
|
||||||
|
for pkg in self.all_package_versions:
|
||||||
|
for tag in pkg.tags:
|
||||||
|
self.all_pkgs_tags_to_version[tag] = pkg
|
||||||
|
logger.info(
|
||||||
|
f"Located {len(self.all_package_versions)} versions of package {self.package_name}",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.decide_what_tags_to_keep()
|
||||||
|
|
||||||
|
def clean(self):
|
||||||
|
for tag_to_delete in self.tags_to_delete:
|
||||||
|
package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
|
||||||
|
|
||||||
|
if self.actually_delete:
|
||||||
|
logger.info(
|
||||||
|
f"Deleting {tag_to_delete} (id {package_version_info.id})",
|
||||||
|
)
|
||||||
|
self.package_api.delete_package_version(
|
||||||
|
package_version_info,
|
||||||
|
)
|
||||||
|
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
f"Would delete {tag_to_delete} (id {package_version_info.id})",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info("No tags to delete")
|
||||||
|
|
||||||
|
def clean_untagged(self, is_manifest_image: bool):
|
||||||
|
def _clean_untagged_manifest():
|
||||||
|
"""
|
||||||
|
|
||||||
|
Handles the deletion of untagged images, but where the package is a manifest, ie a multi
|
||||||
|
arch image, which means some "untagged" images need to exist still.
|
||||||
|
|
||||||
|
Ok, bear with me, these are annoying.
|
||||||
|
|
||||||
|
Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
|
||||||
|
These images are untagged, but pointed to, and so should not be removed (or every pull fails).
|
||||||
|
|
||||||
|
So for each image getting kept, parse the manifest to find the digest(s) it points to. Then
|
||||||
|
remove those from the list of untagged images. The final result is the untagged, not pointed to
|
||||||
|
version which should be safe to remove.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
|
||||||
|
amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
|
||||||
|
armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
|
||||||
|
arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
|
||||||
|
each of which appears as untagged image, but isn't really.
|
||||||
|
|
||||||
|
So from the list of untagged packages, remove those digests. Once all tags which
|
||||||
|
are being kept are checked, the remaining untagged packages are actually untagged
|
||||||
|
with no referrals in a manifest to them.
|
||||||
|
"""
|
||||||
|
# Simplify the untagged data, mapping name (which is a digest) to the version
|
||||||
|
# At the moment, these are the images which APPEAR untagged.
|
||||||
|
untagged_versions = {}
|
||||||
|
for x in self.all_package_versions:
|
||||||
|
if x.untagged:
|
||||||
|
untagged_versions[x.name] = x
|
||||||
|
|
||||||
|
skips = 0
|
||||||
|
|
||||||
|
# Parse manifests to locate digests pointed to
|
||||||
|
for tag in sorted(self.tags_to_keep):
|
||||||
|
full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
|
||||||
|
logger.info(f"Checking manifest for {full_name}")
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(
|
||||||
|
[
|
||||||
|
shutil.which("docker"),
|
||||||
|
"manifest",
|
||||||
|
"inspect",
|
||||||
|
full_name,
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest_list = json.loads(proc.stdout)
|
||||||
|
for manifest_data in manifest_list["manifests"]:
|
||||||
|
manifest = DockerManifest2(manifest_data)
|
||||||
|
|
||||||
|
if manifest.digest in untagged_versions:
|
||||||
|
logger.info(
|
||||||
|
f"Skipping deletion of {manifest.digest},"
|
||||||
|
f" referred to by {full_name}"
|
||||||
|
f" for {manifest.platform}",
|
||||||
|
)
|
||||||
|
del untagged_versions[manifest.digest]
|
||||||
|
skips += 1
|
||||||
|
|
||||||
|
except Exception as err:
|
||||||
|
self.actually_delete = False
|
||||||
|
logger.exception(err)
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"Skipping deletion of {skips} packages referred to by a manifest",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Delete the untagged and not pointed at packages
|
||||||
|
logger.info(f"Deleting untagged packages of {self.package_name}")
|
||||||
|
for to_delete_name in untagged_versions:
|
||||||
|
to_delete_version = untagged_versions[to_delete_name]
|
||||||
|
|
||||||
|
if self.actually_delete:
|
||||||
|
logger.info(
|
||||||
|
f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
|
||||||
|
)
|
||||||
|
self.package_api.delete_package_version(
|
||||||
|
to_delete_version,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
f"Would delete {to_delete_name} (id {to_delete_version.id})",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _clean_untagged_non_manifest():
|
||||||
|
# If the package is not a multi-arch manifest, images without tags are safe to delete.
|
||||||
|
# They are not referred to by anything. This will leave all with at least 1 tag
|
||||||
|
|
||||||
|
for package in self.all_package_versions:
|
||||||
|
if package.untagged:
|
||||||
|
if self.actually_delete:
|
||||||
|
logger.info(
|
||||||
|
f"Deleting id {package.id} named {package.name}",
|
||||||
|
)
|
||||||
|
self.package_api.delete_package_version(
|
||||||
|
package,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
f"Would delete {package.name} (id {package.id})",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
f"Not deleting tag {package.tags[0]} of package {self.package_name}",
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Beginning untagged image cleaning")
|
||||||
|
|
||||||
|
if is_manifest_image:
|
||||||
|
_clean_untagged_manifest()
|
||||||
|
else:
|
||||||
|
_clean_untagged_non_manifest()
|
||||||
|
|
||||||
|
def decide_what_tags_to_keep(self):
|
||||||
|
# By default, keep anything which is tagged
|
||||||
|
self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
|
||||||
|
|
||||||
|
|
||||||
|
class MainImageTagsCleaner(RegistryTagsCleaner):
|
||||||
|
def decide_what_tags_to_keep(self):
|
||||||
|
|
||||||
|
# Locate the feature branches
|
||||||
|
feature_branches = {}
|
||||||
|
for branch in self.branch_api.get_branches(
|
||||||
|
owner=self.repo_owner,
|
||||||
|
repo=self.repo_name,
|
||||||
|
):
|
||||||
|
if branch.name.startswith("feature-"):
|
||||||
|
logger.debug(f"Found feature branch {branch.name}")
|
||||||
|
feature_branches[branch.name] = branch
|
||||||
|
|
||||||
|
logger.info(f"Located {len(feature_branches)} feature branches")
|
||||||
|
|
||||||
|
# Filter to packages which are tagged with feature-*
|
||||||
|
packages_tagged_feature: List[ContainerPackage] = []
|
||||||
|
for package in self.all_package_versions:
|
||||||
|
if package.tag_matches("feature-"):
|
||||||
|
packages_tagged_feature.append(package)
|
||||||
|
|
||||||
|
# Map tags like "feature-xyz" to a ContainerPackage
|
||||||
|
feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
|
||||||
|
for pkg in packages_tagged_feature:
|
||||||
|
for tag in pkg.tags:
|
||||||
|
feature_pkgs_tags_to_versions[tag] = pkg
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f'Located {len(feature_pkgs_tags_to_versions)} versions of package {self.package_name} tagged "feature-"',
|
||||||
|
)
|
||||||
|
|
||||||
|
# All the feature tags minus all the feature branches leaves us feature tags
|
||||||
|
# with no corresponding branch
|
||||||
|
self.tags_to_delete = list(
|
||||||
|
set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
|
||||||
|
)
|
||||||
|
|
||||||
|
# All the tags minus the set of going to be deleted tags leaves us the
|
||||||
|
# tags which will be kept around
|
||||||
|
self.tags_to_keep = list(
|
||||||
|
set(self.all_pkgs_tags_to_version.keys()) - set(self.tags_to_delete),
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
f"Located {len(self.tags_to_delete)} versions of package {self.package_name} to delete",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class LibraryTagsCleaner(RegistryTagsCleaner):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _main():
|
def _main():
|
||||||
parser = ArgumentParser(
|
parser = ArgumentParser(
|
||||||
description="Using the GitHub API locate and optionally delete container"
|
description="Using the GitHub API locate and optionally delete container"
|
||||||
@ -100,190 +339,29 @@ def _main():
|
|||||||
# Note: Only relevant to the main application, but simpler to
|
# Note: Only relevant to the main application, but simpler to
|
||||||
# leave in for all packages
|
# leave in for all packages
|
||||||
with GithubBranchApi(gh_token) as branch_api:
|
with GithubBranchApi(gh_token) as branch_api:
|
||||||
feature_branches = {}
|
with GithubContainerRegistryApi(gh_token, repo_owner) as container_api:
|
||||||
for branch in branch_api.get_branches(
|
if args.package in {"paperless-ngx", "paperless-ngx/builder/cache/app"}:
|
||||||
repo=repo,
|
cleaner = MainImageTagsCleaner(
|
||||||
):
|
args.package,
|
||||||
if branch.name.startswith("feature-"):
|
repo_owner,
|
||||||
logger.debug(f"Found feature branch {branch.name}")
|
repo,
|
||||||
feature_branches[branch.name] = branch
|
container_api,
|
||||||
|
branch_api,
|
||||||
logger.info(f"Located {len(feature_branches)} feature branches")
|
|
||||||
|
|
||||||
with GithubContainerRegistryApi(gh_token, repo_owner) as container_api:
|
|
||||||
# Get the information about all versions of the given package
|
|
||||||
all_package_versions: List[
|
|
||||||
ContainerPackage
|
|
||||||
] = container_api.get_package_versions(args.package)
|
|
||||||
|
|
||||||
all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
|
|
||||||
for pkg in all_package_versions:
|
|
||||||
for tag in pkg.tags:
|
|
||||||
all_pkgs_tags_to_version[tag] = pkg
|
|
||||||
logger.info(
|
|
||||||
f"Located {len(all_package_versions)} versions of package {args.package}",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Filter to packages which are tagged with feature-*
|
|
||||||
packages_tagged_feature: List[ContainerPackage] = []
|
|
||||||
for package in all_package_versions:
|
|
||||||
if package.tag_matches("feature-"):
|
|
||||||
packages_tagged_feature.append(package)
|
|
||||||
|
|
||||||
feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
|
|
||||||
for pkg in packages_tagged_feature:
|
|
||||||
for tag in pkg.tags:
|
|
||||||
feature_pkgs_tags_to_versions[tag] = pkg
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
f'Located {len(feature_pkgs_tags_to_versions)} versions of package {args.package} tagged "feature-"',
|
|
||||||
)
|
|
||||||
|
|
||||||
# All the feature tags minus all the feature branches leaves us feature tags
|
|
||||||
# with no corresponding branch
|
|
||||||
tags_to_delete = list(
|
|
||||||
set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
|
|
||||||
)
|
|
||||||
|
|
||||||
# All the tags minus the set of going to be deleted tags leaves us the
|
|
||||||
# tags which will be kept around
|
|
||||||
tags_to_keep = list(
|
|
||||||
set(all_pkgs_tags_to_version.keys()) - set(tags_to_delete),
|
|
||||||
)
|
|
||||||
logger.info(
|
|
||||||
f"Located {len(tags_to_delete)} versions of package {args.package} to delete",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Delete certain package versions for which no branch existed
|
|
||||||
for tag_to_delete in tags_to_delete:
|
|
||||||
package_version_info = feature_pkgs_tags_to_versions[tag_to_delete]
|
|
||||||
|
|
||||||
if args.delete:
|
|
||||||
logger.info(
|
|
||||||
f"Deleting {tag_to_delete} (id {package_version_info.id})",
|
|
||||||
)
|
)
|
||||||
container_api.delete_package_version(
|
|
||||||
package_version_info,
|
|
||||||
)
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
logger.info(
|
cleaner = LibraryTagsCleaner(
|
||||||
f"Would delete {tag_to_delete} (id {package_version_info.id})",
|
args.package,
|
||||||
|
repo_owner,
|
||||||
|
repo,
|
||||||
|
container_api,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Deal with untagged package versions
|
cleaner.actually_delete = args.delete
|
||||||
if args.untagged:
|
|
||||||
|
|
||||||
logger.info("Handling untagged image packages")
|
cleaner.clean()
|
||||||
|
|
||||||
if not args.is_manifest:
|
cleaner.clean_untagged(args.is_manifest)
|
||||||
# If the package is not a multi-arch manifest, images without tags are safe to delete.
|
|
||||||
# They are not referred to by anything. This will leave all with at least 1 tag
|
|
||||||
|
|
||||||
for package in all_package_versions:
|
|
||||||
if package.untagged:
|
|
||||||
if args.delete:
|
|
||||||
logger.info(
|
|
||||||
f"Deleting id {package.id} named {package.name}",
|
|
||||||
)
|
|
||||||
container_api.delete_package_version(
|
|
||||||
package,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.info(
|
|
||||||
f"Would delete {package.name} (id {package.id})",
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.info(
|
|
||||||
f"Not deleting tag {package.tags[0]} of package {args.package}",
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
|
|
||||||
"""
|
|
||||||
Ok, bear with me, these are annoying.
|
|
||||||
|
|
||||||
Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
|
|
||||||
These images are untagged, but pointed to, and so should not be removed (or every pull fails).
|
|
||||||
|
|
||||||
So for each image getting kept, parse the manifest to find the digest(s) it points to. Then
|
|
||||||
remove those from the list of untagged images. The final result is the untagged, not pointed to
|
|
||||||
version which should be safe to remove.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
|
|
||||||
amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
|
|
||||||
armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
|
|
||||||
arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
|
|
||||||
each of which appears as untagged image, but isn't really.
|
|
||||||
|
|
||||||
So from the list of untagged packages, remove those digests. Once all tags which
|
|
||||||
are being kept are checked, the remaining untagged packages are actually untagged
|
|
||||||
with no referrals in a manifest to them.
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Simplify the untagged data, mapping name (which is a digest) to the version
|
|
||||||
untagged_versions = {}
|
|
||||||
for x in all_package_versions:
|
|
||||||
if x.untagged:
|
|
||||||
untagged_versions[x.name] = x
|
|
||||||
|
|
||||||
skips = 0
|
|
||||||
# Extra security to not delete on an unexpected error
|
|
||||||
actually_delete = True
|
|
||||||
|
|
||||||
# Parse manifests to locate digests pointed to
|
|
||||||
for tag in sorted(tags_to_keep):
|
|
||||||
full_name = f"ghcr.io/{repo_owner}/{args.package}:{tag}"
|
|
||||||
logger.info(f"Checking manifest for {full_name}")
|
|
||||||
try:
|
|
||||||
proc = subprocess.run(
|
|
||||||
[
|
|
||||||
shutil.which("docker"),
|
|
||||||
"manifest",
|
|
||||||
"inspect",
|
|
||||||
full_name,
|
|
||||||
],
|
|
||||||
capture_output=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
manifest_list = json.loads(proc.stdout)
|
|
||||||
for manifest_data in manifest_list["manifests"]:
|
|
||||||
manifest = DockerManifest2(manifest_data)
|
|
||||||
|
|
||||||
if manifest.digest in untagged_versions:
|
|
||||||
logger.debug(
|
|
||||||
f"Skipping deletion of {manifest.digest}, referred to by {full_name} for {manifest.platform}",
|
|
||||||
)
|
|
||||||
del untagged_versions[manifest.digest]
|
|
||||||
skips += 1
|
|
||||||
|
|
||||||
except Exception as err:
|
|
||||||
actually_delete = False
|
|
||||||
logger.exception(err)
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
f"Skipping deletion of {skips} packages referred to by a manifest",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Step 3.3 - Delete the untagged and not pointed at packages
|
|
||||||
logger.info(f"Deleting untagged packages of {args.package}")
|
|
||||||
for to_delete_name in untagged_versions:
|
|
||||||
to_delete_version = untagged_versions[to_delete_name]
|
|
||||||
|
|
||||||
if args.delete and actually_delete:
|
|
||||||
logger.info(
|
|
||||||
f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
|
|
||||||
)
|
|
||||||
container_api.delete_package_version(
|
|
||||||
to_delete_version,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.info(
|
|
||||||
f"Would delete {to_delete_name} (id {to_delete_version.id})",
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.info("Leaving untagged images untouched")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
5
.github/scripts/common.py
vendored
5
.github/scripts/common.py
vendored
@ -29,6 +29,11 @@ def get_cache_image_tag(
|
|||||||
|
|
||||||
|
|
||||||
def get_log_level(args) -> int:
|
def get_log_level(args) -> int:
|
||||||
|
"""
|
||||||
|
Returns a logging level, based
|
||||||
|
:param args:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
levels = {
|
levels = {
|
||||||
"critical": logging.CRITICAL,
|
"critical": logging.CRITICAL,
|
||||||
"error": logging.ERROR,
|
"error": logging.ERROR,
|
||||||
|
54
.github/scripts/github.py
vendored
54
.github/scripts/github.py
vendored
@ -113,14 +113,14 @@ class GithubBranchApi(_GithubApiBase):
|
|||||||
def __init__(self, token: str) -> None:
|
def __init__(self, token: str) -> None:
|
||||||
super().__init__(token)
|
super().__init__(token)
|
||||||
|
|
||||||
self._ENDPOINT = "https://api.github.com/repos/{REPO}/branches"
|
self._ENDPOINT = "https://api.github.com/repos/{OWNER}/{REPO}/branches"
|
||||||
|
|
||||||
def get_branches(self, repo: str) -> List[GithubBranch]:
|
def get_branches(self, owner: str, repo: str) -> List[GithubBranch]:
|
||||||
"""
|
"""
|
||||||
Returns all current branches of the given repository owned by the given
|
Returns all current branches of the given repository owned by the given
|
||||||
owner or organization.
|
owner or organization.
|
||||||
"""
|
"""
|
||||||
endpoint = self._ENDPOINT.format(REPO=repo)
|
endpoint = self._ENDPOINT.format(OWNER=owner, REPO=repo)
|
||||||
internal_data = self._read_all_pages(endpoint)
|
internal_data = self._read_all_pages(endpoint)
|
||||||
return [GithubBranch(branch) for branch in internal_data]
|
return [GithubBranch(branch) for branch in internal_data]
|
||||||
|
|
||||||
@ -189,8 +189,11 @@ class GithubContainerRegistryApi(_GithubApiBase):
|
|||||||
self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
|
self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
|
||||||
# https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
|
# https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
|
||||||
self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
|
self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
|
||||||
|
self._PACKAGE_VERSION_RESTORE_ENDPOINT = (
|
||||||
|
f"{self._PACKAGE_VERSION_DELETE_ENDPOINT}/restore"
|
||||||
|
)
|
||||||
|
|
||||||
def get_package_versions(
|
def get_active_package_versions(
|
||||||
self,
|
self,
|
||||||
package_name: str,
|
package_name: str,
|
||||||
) -> List[ContainerPackage]:
|
) -> List[ContainerPackage]:
|
||||||
@ -216,6 +219,30 @@ class GithubContainerRegistryApi(_GithubApiBase):
|
|||||||
|
|
||||||
return pkgs
|
return pkgs
|
||||||
|
|
||||||
|
def get_deleted_package_versions(
|
||||||
|
self,
|
||||||
|
package_name: str,
|
||||||
|
) -> List[ContainerPackage]:
|
||||||
|
package_type: str = "container"
|
||||||
|
# Need to quote this for slashes in the name
|
||||||
|
package_name = urllib.parse.quote(package_name, safe="")
|
||||||
|
|
||||||
|
endpoint = (
|
||||||
|
self._PACKAGES_VERSIONS_ENDPOINT.format(
|
||||||
|
ORG=self._owner_or_org,
|
||||||
|
PACKAGE_TYPE=package_type,
|
||||||
|
PACKAGE_NAME=package_name,
|
||||||
|
)
|
||||||
|
+ "?state=deleted"
|
||||||
|
)
|
||||||
|
|
||||||
|
pkgs = []
|
||||||
|
|
||||||
|
for data in self._read_all_pages(endpoint):
|
||||||
|
pkgs.append(ContainerPackage(data))
|
||||||
|
|
||||||
|
return pkgs
|
||||||
|
|
||||||
def delete_package_version(self, package_data: ContainerPackage):
|
def delete_package_version(self, package_data: ContainerPackage):
|
||||||
"""
|
"""
|
||||||
Deletes the given package version from the GHCR
|
Deletes the given package version from the GHCR
|
||||||
@ -225,3 +252,22 @@ class GithubContainerRegistryApi(_GithubApiBase):
|
|||||||
logger.warning(
|
logger.warning(
|
||||||
f"Request to delete {package_data.url} returned HTTP {resp.status_code}",
|
f"Request to delete {package_data.url} returned HTTP {resp.status_code}",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def restore_package_version(
|
||||||
|
self,
|
||||||
|
package_name: str,
|
||||||
|
package_data: ContainerPackage,
|
||||||
|
):
|
||||||
|
package_type: str = "container"
|
||||||
|
endpoint = self._PACKAGE_VERSION_RESTORE_ENDPOINT.format(
|
||||||
|
ORG=self._owner_or_org,
|
||||||
|
PACKAGE_TYPE=package_type,
|
||||||
|
PACKAGE_NAME=package_name,
|
||||||
|
PACKAGE_VERSION_ID=package_data.id,
|
||||||
|
)
|
||||||
|
|
||||||
|
resp = self._session.post(endpoint)
|
||||||
|
if resp.status_code != 204:
|
||||||
|
logger.warning(
|
||||||
|
f"Request to delete {endpoint} returned HTTP {resp.status_code}",
|
||||||
|
)
|
||||||
|
89
.github/workflows/cleanup-tags.yml
vendored
89
.github/workflows/cleanup-tags.yml
vendored
@ -15,7 +15,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
paths:
|
paths:
|
||||||
- ".github/workflows/cleanup-tags.yml"
|
- ".github/workflows/cleanup-tags.yml"
|
||||||
- ".github/scripts/cleanup-tags.py"
|
- ".github/scripts/cleanup-images.py"
|
||||||
- ".github/scripts/github.py"
|
- ".github/scripts/github.py"
|
||||||
- ".github/scripts/common.py"
|
- ".github/scripts/common.py"
|
||||||
|
|
||||||
@ -24,9 +24,26 @@ concurrency:
|
|||||||
cancel-in-progress: false
|
cancel-in-progress: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
cleanup:
|
cleanup-images:
|
||||||
name: Cleanup Image Tags
|
name: Cleanup Image Tags for ${{ matrix.primary-name }}
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- primary-name: "paperless-ngx"
|
||||||
|
cache-name: "paperless-ngx/builder/cache/app"
|
||||||
|
|
||||||
|
- primary-name: "paperless-ngx/builder/qpdf"
|
||||||
|
cache-name: "paperless-ngx/builder/cache/qpdf"
|
||||||
|
|
||||||
|
- primary-name: "paperless-ngx/builder/pikepdf"
|
||||||
|
cache-name: "paperless-ngx/builder/cache/pikepdf"
|
||||||
|
|
||||||
|
- primary-name: "paperless-ngx/builder/jbig2enc"
|
||||||
|
cache-name: "paperless-ngx/builder/cache/jbig2enc"
|
||||||
|
|
||||||
|
- primary-name: "paperless-ngx/builder/psycopg2"
|
||||||
|
cache-name: "paperless-ngx/builder/cache/psycopg2"
|
||||||
env:
|
env:
|
||||||
# Requires a personal access token with the OAuth scope delete:packages
|
# Requires a personal access token with the OAuth scope delete:packages
|
||||||
TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}
|
TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}
|
||||||
@ -50,63 +67,29 @@ jobs:
|
|||||||
name: Install requests
|
name: Install requests
|
||||||
run: |
|
run: |
|
||||||
python -m pip install requests
|
python -m pip install requests
|
||||||
# Clean up primary packages
|
|
||||||
-
|
|
||||||
name: Cleanup for package "paperless-ngx"
|
|
||||||
if: "${{ env.TOKEN != '' }}"
|
|
||||||
run: |
|
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx"
|
|
||||||
-
|
|
||||||
name: Cleanup for package "qpdf"
|
|
||||||
if: "${{ env.TOKEN != '' }}"
|
|
||||||
run: |
|
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/qpdf"
|
|
||||||
-
|
|
||||||
name: Cleanup for package "pikepdf"
|
|
||||||
if: "${{ env.TOKEN != '' }}"
|
|
||||||
run: |
|
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/pikepdf"
|
|
||||||
-
|
|
||||||
name: Cleanup for package "jbig2enc"
|
|
||||||
if: "${{ env.TOKEN != '' }}"
|
|
||||||
run: |
|
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/jbig2enc"
|
|
||||||
-
|
|
||||||
name: Cleanup for package "psycopg2"
|
|
||||||
if: "${{ env.TOKEN != '' }}"
|
|
||||||
run: |
|
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/psycopg2"
|
|
||||||
#
|
#
|
||||||
# Clean up registry cache packages
|
# Clean up primary package
|
||||||
#
|
#
|
||||||
-
|
-
|
||||||
name: Cleanup for package "builder/cache/app"
|
name: Cleanup for package "${{ matrix.primary-name }}"
|
||||||
if: "${{ env.TOKEN != '' }}"
|
if: "${{ env.TOKEN != '' }}"
|
||||||
run: |
|
run: |
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/app"
|
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-images.py --untagged --is-manifest "${{ matrix.primary-name }}"
|
||||||
|
#
|
||||||
|
# Clean up registry cache package
|
||||||
|
#
|
||||||
-
|
-
|
||||||
name: Cleanup for package "builder/cache/qpdf"
|
name: Cleanup for package "${{ matrix.cache-name }}"
|
||||||
if: "${{ env.TOKEN != '' }}"
|
if: "${{ env.TOKEN != '' }}"
|
||||||
run: |
|
run: |
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/qpdf"
|
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-images.py --untagged "${{ matrix.cache-name }}"
|
||||||
-
|
#
|
||||||
name: Cleanup for package "builder/cache/psycopg2"
|
# Verify tags which are left still pull
|
||||||
if: "${{ env.TOKEN != '' }}"
|
#
|
||||||
run: |
|
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/psycopg2"
|
|
||||||
-
|
|
||||||
name: Cleanup for package "builder/cache/jbig2enc"
|
|
||||||
if: "${{ env.TOKEN != '' }}"
|
|
||||||
run: |
|
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/jbig2enc"
|
|
||||||
-
|
|
||||||
name: Cleanup for package "builder/cache/pikepdf"
|
|
||||||
if: "${{ env.TOKEN != '' }}"
|
|
||||||
run: |
|
|
||||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/pikepdf"
|
|
||||||
-
|
-
|
||||||
name: Check all tags still pull
|
name: Check all tags still pull
|
||||||
run: |
|
run: |
|
||||||
ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
|
ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }')
|
||||||
echo "Pulling all tags of ghcr.io/${ghcr_name}"
|
echo "Pulling all tags of ${ghcr_name}"
|
||||||
docker pull --quiet --all-tags ghcr.io/${ghcr_name}
|
docker pull --quiet --all-tags ${ghcr_name}
|
||||||
|
docker image list
|
||||||
|
Loading…
x
Reference in New Issue
Block a user