mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-04-02 13:45:10 -05:00
Merge branch 'dev' of github.com:paperless-ngx/paperless-ngx into dev
This commit is contained in:
Michael Shamoon
commit
99db828d49
462
.github/scripts/cleanup-tags.py
vendored
462
.github/scripts/cleanup-tags.py
vendored
@ -8,6 +8,7 @@ from argparse import ArgumentParser
|
||||
from typing import Dict
|
||||
from typing import Final
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
|
||||
from common import get_log_level
|
||||
from github import ContainerPackage
|
||||
@ -26,7 +27,7 @@ class DockerManifest2:
|
||||
|
||||
def __init__(self, data: Dict) -> None:
|
||||
self._data = data
|
||||
# This is the sha256: digest string. Corresponds to Github API name
|
||||
# This is the sha256: digest string. Corresponds to GitHub API name
|
||||
# if the package is an untagged package
|
||||
self.digest = self._data["digest"]
|
||||
platform_data_os = self._data["platform"]["os"]
|
||||
@ -38,6 +39,269 @@ class DockerManifest2:
|
||||
self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
|
||||
|
||||
|
||||
class RegistryTagsCleaner:
|
||||
"""
|
||||
This is the base class for the image registry cleaning. Given a package
|
||||
name, it will keep all images which are tagged and all untagged images
|
||||
referred to by a manifest. This results in only images which have been untagged
|
||||
and cannot be referenced except by their SHA in being removed. None of these
|
||||
images should be referenced, so it is fine to delete them.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
package_name: str,
|
||||
repo_owner: str,
|
||||
repo_name: str,
|
||||
package_api: GithubContainerRegistryApi,
|
||||
branch_api: Optional[GithubBranchApi],
|
||||
):
|
||||
self.actually_delete = False
|
||||
self.package_api = package_api
|
||||
self.branch_api = branch_api
|
||||
self.package_name = package_name
|
||||
self.repo_owner = repo_owner
|
||||
self.repo_name = repo_name
|
||||
self.tags_to_delete: List[str] = []
|
||||
self.tags_to_keep: List[str] = []
|
||||
|
||||
# Get the information about all versions of the given package
|
||||
# These are active, not deleted, the default returned from the API
|
||||
self.all_package_versions = self.package_api.get_active_package_versions(
|
||||
self.package_name,
|
||||
)
|
||||
|
||||
# Get a mapping from a tag like "1.7.0" or "feature-xyz" to the ContainerPackage
|
||||
# tagged with it. It makes certain lookups easy
|
||||
self.all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
|
||||
for pkg in self.all_package_versions:
|
||||
for tag in pkg.tags:
|
||||
self.all_pkgs_tags_to_version[tag] = pkg
|
||||
logger.info(
|
||||
f"Located {len(self.all_package_versions)} versions of package {self.package_name}",
|
||||
)
|
||||
|
||||
self.decide_what_tags_to_keep()
|
||||
|
||||
def clean(self):
|
||||
"""
|
||||
This method will delete image versions, based on the selected tags to delete
|
||||
"""
|
||||
for tag_to_delete in self.tags_to_delete:
|
||||
package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
|
||||
|
||||
if self.actually_delete:
|
||||
logger.info(
|
||||
f"Deleting {tag_to_delete} (id {package_version_info.id})",
|
||||
)
|
||||
self.package_api.delete_package_version(
|
||||
package_version_info,
|
||||
)
|
||||
|
||||
else:
|
||||
logger.info(
|
||||
f"Would delete {tag_to_delete} (id {package_version_info.id})",
|
||||
)
|
||||
else:
|
||||
logger.info("No tags to delete")
|
||||
|
||||
def clean_untagged(self, is_manifest_image: bool):
|
||||
"""
|
||||
This method will delete untagged images, that is those which are not named. It
|
||||
handles if the image tag is actually a manifest, which points to images that look otherwise
|
||||
untagged.
|
||||
"""
|
||||
|
||||
def _clean_untagged_manifest():
|
||||
"""
|
||||
|
||||
Handles the deletion of untagged images, but where the package is a manifest, ie a multi
|
||||
arch image, which means some "untagged" images need to exist still.
|
||||
|
||||
Ok, bear with me, these are annoying.
|
||||
|
||||
Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
|
||||
These images are untagged, but pointed to, and so should not be removed (or every pull fails).
|
||||
|
||||
So for each image getting kept, parse the manifest to find the digest(s) it points to. Then
|
||||
remove those from the list of untagged images. The final result is the untagged, not pointed to
|
||||
version which should be safe to remove.
|
||||
|
||||
Example:
|
||||
Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
|
||||
amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
|
||||
armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
|
||||
arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
|
||||
each of which appears as untagged image, but isn't really.
|
||||
|
||||
So from the list of untagged packages, remove those digests. Once all tags which
|
||||
are being kept are checked, the remaining untagged packages are actually untagged
|
||||
with no referrals in a manifest to them.
|
||||
"""
|
||||
# Simplify the untagged data, mapping name (which is a digest) to the version
|
||||
# At the moment, these are the images which APPEAR untagged.
|
||||
untagged_versions = {}
|
||||
for x in self.all_package_versions:
|
||||
if x.untagged:
|
||||
untagged_versions[x.name] = x
|
||||
|
||||
skips = 0
|
||||
|
||||
# Parse manifests to locate digests pointed to
|
||||
for tag in sorted(self.tags_to_keep):
|
||||
full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
|
||||
logger.info(f"Checking manifest for {full_name}")
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
shutil.which("docker"),
|
||||
"manifest",
|
||||
"inspect",
|
||||
full_name,
|
||||
],
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
manifest_list = json.loads(proc.stdout)
|
||||
for manifest_data in manifest_list["manifests"]:
|
||||
manifest = DockerManifest2(manifest_data)
|
||||
|
||||
if manifest.digest in untagged_versions:
|
||||
logger.info(
|
||||
f"Skipping deletion of {manifest.digest},"
|
||||
f" referred to by {full_name}"
|
||||
f" for {manifest.platform}",
|
||||
)
|
||||
del untagged_versions[manifest.digest]
|
||||
skips += 1
|
||||
|
||||
except Exception as err:
|
||||
self.actually_delete = False
|
||||
logger.exception(err)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
f"Skipping deletion of {skips} packages referred to by a manifest",
|
||||
)
|
||||
|
||||
# Delete the untagged and not pointed at packages
|
||||
logger.info(f"Deleting untagged packages of {self.package_name}")
|
||||
for to_delete_name in untagged_versions:
|
||||
to_delete_version = untagged_versions[to_delete_name]
|
||||
|
||||
if self.actually_delete:
|
||||
logger.info(
|
||||
f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
|
||||
)
|
||||
self.package_api.delete_package_version(
|
||||
to_delete_version,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Would delete {to_delete_name} (id {to_delete_version.id})",
|
||||
)
|
||||
|
||||
def _clean_untagged_non_manifest():
|
||||
"""
|
||||
If the package is not a multi-arch manifest, images without tags are safe to delete.
|
||||
"""
|
||||
|
||||
for package in self.all_package_versions:
|
||||
if package.untagged:
|
||||
if self.actually_delete:
|
||||
logger.info(
|
||||
f"Deleting id {package.id} named {package.name}",
|
||||
)
|
||||
self.package_api.delete_package_version(
|
||||
package,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Would delete {package.name} (id {package.id})",
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Not deleting tag {package.tags[0]} of package {self.package_name}",
|
||||
)
|
||||
|
||||
logger.info("Beginning untagged image cleaning")
|
||||
|
||||
if is_manifest_image:
|
||||
_clean_untagged_manifest()
|
||||
else:
|
||||
_clean_untagged_non_manifest()
|
||||
|
||||
def decide_what_tags_to_keep(self):
|
||||
"""
|
||||
This method holds the logic to delete what tags to keep and there fore
|
||||
what tags to delete.
|
||||
|
||||
By default, any image with at least 1 tag will be kept
|
||||
"""
|
||||
# By default, keep anything which is tagged
|
||||
self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
|
||||
|
||||
|
||||
class MainImageTagsCleaner(RegistryTagsCleaner):
|
||||
def decide_what_tags_to_keep(self):
|
||||
"""
|
||||
Overrides the default logic for deciding what images to keep. Images tagged as "feature-"
|
||||
will be removed, if the corresponding branch no longer exists.
|
||||
"""
|
||||
|
||||
# Locate the feature branches
|
||||
feature_branches = {}
|
||||
for branch in self.branch_api.get_branches(
|
||||
owner=self.repo_owner,
|
||||
repo=self.repo_name,
|
||||
):
|
||||
if branch.name.startswith("feature-"):
|
||||
logger.debug(f"Found feature branch {branch.name}")
|
||||
feature_branches[branch.name] = branch
|
||||
|
||||
logger.info(f"Located {len(feature_branches)} feature branches")
|
||||
|
||||
# Filter to packages which are tagged with feature-*
|
||||
packages_tagged_feature: List[ContainerPackage] = []
|
||||
for package in self.all_package_versions:
|
||||
if package.tag_matches("feature-"):
|
||||
packages_tagged_feature.append(package)
|
||||
|
||||
# Map tags like "feature-xyz" to a ContainerPackage
|
||||
feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
|
||||
for pkg in packages_tagged_feature:
|
||||
for tag in pkg.tags:
|
||||
feature_pkgs_tags_to_versions[tag] = pkg
|
||||
|
||||
logger.info(
|
||||
f'Located {len(feature_pkgs_tags_to_versions)} versions of package {self.package_name} tagged "feature-"',
|
||||
)
|
||||
|
||||
# All the feature tags minus all the feature branches leaves us feature tags
|
||||
# with no corresponding branch
|
||||
self.tags_to_delete = list(
|
||||
set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
|
||||
)
|
||||
|
||||
# All the tags minus the set of going to be deleted tags leaves us the
|
||||
# tags which will be kept around
|
||||
self.tags_to_keep = list(
|
||||
set(self.all_pkgs_tags_to_version.keys()) - set(self.tags_to_delete),
|
||||
)
|
||||
logger.info(
|
||||
f"Located {len(self.tags_to_delete)} versions of package {self.package_name} to delete",
|
||||
)
|
||||
|
||||
|
||||
class LibraryTagsCleaner(RegistryTagsCleaner):
|
||||
"""
|
||||
Exists for the off change that someday, the installer library images
|
||||
will need their own logic
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def _main():
|
||||
parser = ArgumentParser(
|
||||
description="Using the GitHub API locate and optionally delete container"
|
||||
@ -100,190 +364,32 @@ def _main():
|
||||
# Note: Only relevant to the main application, but simpler to
|
||||
# leave in for all packages
|
||||
with GithubBranchApi(gh_token) as branch_api:
|
||||
feature_branches = {}
|
||||
for branch in branch_api.get_branches(
|
||||
repo=repo,
|
||||
):
|
||||
if branch.name.startswith("feature-"):
|
||||
logger.debug(f"Found feature branch {branch.name}")
|
||||
feature_branches[branch.name] = branch
|
||||
|
||||
logger.info(f"Located {len(feature_branches)} feature branches")
|
||||
|
||||
with GithubContainerRegistryApi(gh_token, repo_owner) as container_api:
|
||||
# Get the information about all versions of the given package
|
||||
all_package_versions: List[
|
||||
ContainerPackage
|
||||
] = container_api.get_package_versions(args.package)
|
||||
|
||||
all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
|
||||
for pkg in all_package_versions:
|
||||
for tag in pkg.tags:
|
||||
all_pkgs_tags_to_version[tag] = pkg
|
||||
logger.info(
|
||||
f"Located {len(all_package_versions)} versions of package {args.package}",
|
||||
)
|
||||
|
||||
# Filter to packages which are tagged with feature-*
|
||||
packages_tagged_feature: List[ContainerPackage] = []
|
||||
for package in all_package_versions:
|
||||
if package.tag_matches("feature-"):
|
||||
packages_tagged_feature.append(package)
|
||||
|
||||
feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
|
||||
for pkg in packages_tagged_feature:
|
||||
for tag in pkg.tags:
|
||||
feature_pkgs_tags_to_versions[tag] = pkg
|
||||
|
||||
logger.info(
|
||||
f'Located {len(feature_pkgs_tags_to_versions)} versions of package {args.package} tagged "feature-"',
|
||||
)
|
||||
|
||||
# All the feature tags minus all the feature branches leaves us feature tags
|
||||
# with no corresponding branch
|
||||
tags_to_delete = list(
|
||||
set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
|
||||
)
|
||||
|
||||
# All the tags minus the set of going to be deleted tags leaves us the
|
||||
# tags which will be kept around
|
||||
tags_to_keep = list(
|
||||
set(all_pkgs_tags_to_version.keys()) - set(tags_to_delete),
|
||||
)
|
||||
logger.info(
|
||||
f"Located {len(tags_to_delete)} versions of package {args.package} to delete",
|
||||
)
|
||||
|
||||
# Delete certain package versions for which no branch existed
|
||||
for tag_to_delete in tags_to_delete:
|
||||
package_version_info = feature_pkgs_tags_to_versions[tag_to_delete]
|
||||
|
||||
if args.delete:
|
||||
logger.info(
|
||||
f"Deleting {tag_to_delete} (id {package_version_info.id})",
|
||||
)
|
||||
container_api.delete_package_version(
|
||||
package_version_info,
|
||||
)
|
||||
|
||||
else:
|
||||
logger.info(
|
||||
f"Would delete {tag_to_delete} (id {package_version_info.id})",
|
||||
)
|
||||
|
||||
# Deal with untagged package versions
|
||||
if args.untagged:
|
||||
|
||||
logger.info("Handling untagged image packages")
|
||||
|
||||
if not args.is_manifest:
|
||||
# If the package is not a multi-arch manifest, images without tags are safe to delete.
|
||||
# They are not referred to by anything. This will leave all with at least 1 tag
|
||||
|
||||
for package in all_package_versions:
|
||||
if package.untagged:
|
||||
if args.delete:
|
||||
logger.info(
|
||||
f"Deleting id {package.id} named {package.name}",
|
||||
)
|
||||
container_api.delete_package_version(
|
||||
package,
|
||||
if args.package in {"paperless-ngx", "paperless-ngx/builder/cache/app"}:
|
||||
cleaner = MainImageTagsCleaner(
|
||||
args.package,
|
||||
repo_owner,
|
||||
repo,
|
||||
container_api,
|
||||
branch_api,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Would delete {package.name} (id {package.id})",
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Not deleting tag {package.tags[0]} of package {args.package}",
|
||||
)
|
||||
else:
|
||||
|
||||
"""
|
||||
Ok, bear with me, these are annoying.
|
||||
|
||||
Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
|
||||
These images are untagged, but pointed to, and so should not be removed (or every pull fails).
|
||||
|
||||
So for each image getting kept, parse the manifest to find the digest(s) it points to. Then
|
||||
remove those from the list of untagged images. The final result is the untagged, not pointed to
|
||||
version which should be safe to remove.
|
||||
|
||||
Example:
|
||||
Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
|
||||
amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
|
||||
armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
|
||||
arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
|
||||
each of which appears as untagged image, but isn't really.
|
||||
|
||||
So from the list of untagged packages, remove those digests. Once all tags which
|
||||
are being kept are checked, the remaining untagged packages are actually untagged
|
||||
with no referrals in a manifest to them.
|
||||
|
||||
"""
|
||||
|
||||
# Simplify the untagged data, mapping name (which is a digest) to the version
|
||||
untagged_versions = {}
|
||||
for x in all_package_versions:
|
||||
if x.untagged:
|
||||
untagged_versions[x.name] = x
|
||||
|
||||
skips = 0
|
||||
# Extra security to not delete on an unexpected error
|
||||
actually_delete = True
|
||||
|
||||
# Parse manifests to locate digests pointed to
|
||||
for tag in sorted(tags_to_keep):
|
||||
full_name = f"ghcr.io/{repo_owner}/{args.package}:{tag}"
|
||||
logger.info(f"Checking manifest for {full_name}")
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
shutil.which("docker"),
|
||||
"manifest",
|
||||
"inspect",
|
||||
full_name,
|
||||
],
|
||||
capture_output=True,
|
||||
cleaner = LibraryTagsCleaner(
|
||||
args.package,
|
||||
repo_owner,
|
||||
repo,
|
||||
container_api,
|
||||
None,
|
||||
)
|
||||
|
||||
manifest_list = json.loads(proc.stdout)
|
||||
for manifest_data in manifest_list["manifests"]:
|
||||
manifest = DockerManifest2(manifest_data)
|
||||
# Set if actually doing a delete vs dry run
|
||||
cleaner.actually_delete = args.delete
|
||||
|
||||
if manifest.digest in untagged_versions:
|
||||
logger.debug(
|
||||
f"Skipping deletion of {manifest.digest}, referred to by {full_name} for {manifest.platform}",
|
||||
)
|
||||
del untagged_versions[manifest.digest]
|
||||
skips += 1
|
||||
# Clean images with tags
|
||||
cleaner.clean()
|
||||
|
||||
except Exception as err:
|
||||
actually_delete = False
|
||||
logger.exception(err)
|
||||
|
||||
logger.info(
|
||||
f"Skipping deletion of {skips} packages referred to by a manifest",
|
||||
)
|
||||
|
||||
# Step 3.3 - Delete the untagged and not pointed at packages
|
||||
logger.info(f"Deleting untagged packages of {args.package}")
|
||||
for to_delete_name in untagged_versions:
|
||||
to_delete_version = untagged_versions[to_delete_name]
|
||||
|
||||
if args.delete and actually_delete:
|
||||
logger.info(
|
||||
f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
|
||||
)
|
||||
container_api.delete_package_version(
|
||||
to_delete_version,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Would delete {to_delete_name} (id {to_delete_version.id})",
|
||||
)
|
||||
else:
|
||||
logger.info("Leaving untagged images untouched")
|
||||
# Clean images which are untagged
|
||||
cleaner.clean_untagged(args.is_manifest)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
5
.github/scripts/common.py
vendored
5
.github/scripts/common.py
vendored
@ -29,6 +29,11 @@ def get_cache_image_tag(
|
||||
|
||||
|
||||
def get_log_level(args) -> int:
|
||||
"""
|
||||
Returns a logging level, based
|
||||
:param args:
|
||||
:return:
|
||||
"""
|
||||
levels = {
|
||||
"critical": logging.CRITICAL,
|
||||
"error": logging.ERROR,
|
||||
|
||||
54
.github/scripts/github.py
vendored
54
.github/scripts/github.py
vendored
@ -113,14 +113,14 @@ class GithubBranchApi(_GithubApiBase):
|
||||
def __init__(self, token: str) -> None:
|
||||
super().__init__(token)
|
||||
|
||||
self._ENDPOINT = "https://api.github.com/repos/{REPO}/branches"
|
||||
self._ENDPOINT = "https://api.github.com/repos/{OWNER}/{REPO}/branches"
|
||||
|
||||
def get_branches(self, repo: str) -> List[GithubBranch]:
|
||||
def get_branches(self, owner: str, repo: str) -> List[GithubBranch]:
|
||||
"""
|
||||
Returns all current branches of the given repository owned by the given
|
||||
owner or organization.
|
||||
"""
|
||||
endpoint = self._ENDPOINT.format(REPO=repo)
|
||||
endpoint = self._ENDPOINT.format(OWNER=owner, REPO=repo)
|
||||
internal_data = self._read_all_pages(endpoint)
|
||||
return [GithubBranch(branch) for branch in internal_data]
|
||||
|
||||
@ -189,8 +189,11 @@ class GithubContainerRegistryApi(_GithubApiBase):
|
||||
self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
|
||||
# https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
|
||||
self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
|
||||
self._PACKAGE_VERSION_RESTORE_ENDPOINT = (
|
||||
f"{self._PACKAGE_VERSION_DELETE_ENDPOINT}/restore"
|
||||
)
|
||||
|
||||
def get_package_versions(
|
||||
def get_active_package_versions(
|
||||
self,
|
||||
package_name: str,
|
||||
) -> List[ContainerPackage]:
|
||||
@ -216,6 +219,30 @@ class GithubContainerRegistryApi(_GithubApiBase):
|
||||
|
||||
return pkgs
|
||||
|
||||
def get_deleted_package_versions(
|
||||
self,
|
||||
package_name: str,
|
||||
) -> List[ContainerPackage]:
|
||||
package_type: str = "container"
|
||||
# Need to quote this for slashes in the name
|
||||
package_name = urllib.parse.quote(package_name, safe="")
|
||||
|
||||
endpoint = (
|
||||
self._PACKAGES_VERSIONS_ENDPOINT.format(
|
||||
ORG=self._owner_or_org,
|
||||
PACKAGE_TYPE=package_type,
|
||||
PACKAGE_NAME=package_name,
|
||||
)
|
||||
+ "?state=deleted"
|
||||
)
|
||||
|
||||
pkgs = []
|
||||
|
||||
for data in self._read_all_pages(endpoint):
|
||||
pkgs.append(ContainerPackage(data))
|
||||
|
||||
return pkgs
|
||||
|
||||
def delete_package_version(self, package_data: ContainerPackage):
|
||||
"""
|
||||
Deletes the given package version from the GHCR
|
||||
@ -225,3 +252,22 @@ class GithubContainerRegistryApi(_GithubApiBase):
|
||||
logger.warning(
|
||||
f"Request to delete {package_data.url} returned HTTP {resp.status_code}",
|
||||
)
|
||||
|
||||
def restore_package_version(
|
||||
self,
|
||||
package_name: str,
|
||||
package_data: ContainerPackage,
|
||||
):
|
||||
package_type: str = "container"
|
||||
endpoint = self._PACKAGE_VERSION_RESTORE_ENDPOINT.format(
|
||||
ORG=self._owner_or_org,
|
||||
PACKAGE_TYPE=package_type,
|
||||
PACKAGE_NAME=package_name,
|
||||
PACKAGE_VERSION_ID=package_data.id,
|
||||
)
|
||||
|
||||
resp = self._session.post(endpoint)
|
||||
if resp.status_code != 204:
|
||||
logger.warning(
|
||||
f"Request to delete {endpoint} returned HTTP {resp.status_code}",
|
||||
)
|
||||
|
||||
87
.github/workflows/cleanup-tags.yml
vendored
87
.github/workflows/cleanup-tags.yml
vendored
@ -24,9 +24,26 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
cleanup:
|
||||
name: Cleanup Image Tags
|
||||
runs-on: ubuntu-20.04
|
||||
cleanup-images:
|
||||
name: Cleanup Image Tags for ${{ matrix.primary-name }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- primary-name: "paperless-ngx"
|
||||
cache-name: "paperless-ngx/builder/cache/app"
|
||||
|
||||
- primary-name: "paperless-ngx/builder/qpdf"
|
||||
cache-name: "paperless-ngx/builder/cache/qpdf"
|
||||
|
||||
- primary-name: "paperless-ngx/builder/pikepdf"
|
||||
cache-name: "paperless-ngx/builder/cache/pikepdf"
|
||||
|
||||
- primary-name: "paperless-ngx/builder/jbig2enc"
|
||||
cache-name: "paperless-ngx/builder/cache/jbig2enc"
|
||||
|
||||
- primary-name: "paperless-ngx/builder/psycopg2"
|
||||
cache-name: "paperless-ngx/builder/cache/psycopg2"
|
||||
env:
|
||||
# Requires a personal access token with the OAuth scope delete:packages
|
||||
TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}
|
||||
@ -50,63 +67,29 @@ jobs:
|
||||
name: Install requests
|
||||
run: |
|
||||
python -m pip install requests
|
||||
# Clean up primary packages
|
||||
-
|
||||
name: Cleanup for package "paperless-ngx"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx"
|
||||
-
|
||||
name: Cleanup for package "qpdf"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/qpdf"
|
||||
-
|
||||
name: Cleanup for package "pikepdf"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/pikepdf"
|
||||
-
|
||||
name: Cleanup for package "jbig2enc"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/jbig2enc"
|
||||
-
|
||||
name: Cleanup for package "psycopg2"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/psycopg2"
|
||||
#
|
||||
# Clean up registry cache packages
|
||||
# Clean up primary package
|
||||
#
|
||||
-
|
||||
name: Cleanup for package "builder/cache/app"
|
||||
name: Cleanup for package "${{ matrix.primary-name }}"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/app"
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --is-manifest --delete "${{ matrix.primary-name }}"
|
||||
#
|
||||
# Clean up registry cache package
|
||||
#
|
||||
-
|
||||
name: Cleanup for package "builder/cache/qpdf"
|
||||
name: Cleanup for package "${{ matrix.cache-name }}"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/qpdf"
|
||||
-
|
||||
name: Cleanup for package "builder/cache/psycopg2"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/psycopg2"
|
||||
-
|
||||
name: Cleanup for package "builder/cache/jbig2enc"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/jbig2enc"
|
||||
-
|
||||
name: Cleanup for package "builder/cache/pikepdf"
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
run: |
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/pikepdf"
|
||||
python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}"
|
||||
#
|
||||
# Verify tags which are left still pull
|
||||
#
|
||||
-
|
||||
name: Check all tags still pull
|
||||
run: |
|
||||
ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
|
||||
echo "Pulling all tags of ghcr.io/${ghcr_name}"
|
||||
docker pull --quiet --all-tags ghcr.io/${ghcr_name}
|
||||
ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }')
|
||||
echo "Pulling all tags of ${ghcr_name}"
|
||||
docker pull --quiet --all-tags ${ghcr_name}
|
||||
docker image list
|
||||
|
||||
2
Pipfile
2
Pipfile
@ -71,7 +71,7 @@ pytest-django = "*"
|
||||
pytest-env = "*"
|
||||
pytest-sugar = "*"
|
||||
pytest-xdist = "*"
|
||||
sphinx = "~=5.1"
|
||||
sphinx = "~=5.3"
|
||||
sphinx_rtd_theme = "*"
|
||||
tox = "*"
|
||||
black = "*"
|
||||
|
||||
258
Pipfile.lock
generated
258
Pipfile.lock
generated
@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "91a00f3941172f541e45d764727a12280c0be0898ec0e294c6d1283208f9bc92"
|
||||
"sha256": "68ff2e4e4ebbed482cc7d646337309ffd51140748b83d0b60d22dec9926f2ccb"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {},
|
||||
@ -218,7 +218,7 @@
|
||||
"sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845",
|
||||
"sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"
|
||||
],
|
||||
"markers": "python_full_version >= '3.6.0'",
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2.1.1"
|
||||
},
|
||||
"click": {
|
||||
@ -234,7 +234,7 @@
|
||||
"sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667",
|
||||
"sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"
|
||||
],
|
||||
"markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'",
|
||||
"markers": "python_version < '4' and python_full_version >= '3.6.2'",
|
||||
"version": "==0.3.0"
|
||||
},
|
||||
"click-plugins": {
|
||||
@ -562,10 +562,10 @@
|
||||
},
|
||||
"incremental": {
|
||||
"hashes": [
|
||||
"sha256:02f5de5aff48f6b9f665d99d48bfc7ec03b6e3943210de7cfc88856d755d6f57",
|
||||
"sha256:92014aebc6a20b78a8084cdd5645eeaa7f74b8933f70fa3ada2cfbd1e3b54321"
|
||||
"sha256:912feeb5e0f7e0188e6f42241d2f450002e11bbc0937c65865045854c24c0bd0",
|
||||
"sha256:b864a1f30885ee72c5ac2835a761b8fe8aa9c28b9395cacf27286602688d3e51"
|
||||
],
|
||||
"version": "==21.3.0"
|
||||
"version": "==22.10.0"
|
||||
},
|
||||
"inotify-simple": {
|
||||
"hashes": [
|
||||
@ -608,111 +608,111 @@
|
||||
},
|
||||
"levenshtein": {
|
||||
"hashes": [
|
||||
"sha256:02f3914af0d1d45764e981a4c0637cc747b73e482250d251c4f268c969d54bce",
|
||||
"sha256:03fe04ce29cf85b62a4cd03a1fff327ba30b440342f0698884bc98c2382c80ab",
|
||||
"sha256:047b65c59356e59a9f1d90350f057ec2ad17c5bbb25bd18cefdec7318e7a881c",
|
||||
"sha256:04cceb79d9b962c80bfd583ca76364e984c769d8ef47aa56443c4c1d88a16376",
|
||||
"sha256:08de9fe0bb8bef7b06502ed914431bf708fe35aa1690a14df64825ad3a67fd0d",
|
||||
"sha256:0a7ced0ad7f88105519ce2dcca97b95ee97516e8461657ff825c093dc680b8fa",
|
||||
"sha256:0ab47e8c409afd83d4d21ecab0e81bd05930bac2035936d7b670720fa534fa4d",
|
||||
"sha256:0ea3c2585385fe3cda86e5720cd8ab8fb6fa40e0161339cc3c0bfc35e2312a7d",
|
||||
"sha256:114ec9dffdae87dfbf88f9e0f1b7b9333973bce4af0f0cc71537b1ebf098fe02",
|
||||
"sha256:1262330d8a3a358625397269925620d7ded2e1ec531ed49181a84954d5cc9054",
|
||||
"sha256:12d210257911b3ab4c0f83c8de537d2ca2489bac12d7ce860b5ae6e9197f742c",
|
||||
"sha256:131f8ec69ba6fe1c8a78236d9ffbf3cf68d4125134dfcabc3305df2d791384e8",
|
||||
"sha256:1bd77a327200a244f5179fcb3f7674a1588cbf95da0368265d50fc70942c3ec6",
|
||||
"sha256:1d0388fc101e51f19512d1c204202a96d7e069489369ba2ffc31596065156ccf",
|
||||
"sha256:1e4bcd812105890bfda1809d096472fec223da9145aff7f508eb2bc770ca3e00",
|
||||
"sha256:1f5563a21096bee6fadd714e175f6b4f1f06747f8e3aea09f2f747889df11f54",
|
||||
"sha256:20530b025d5d5b118aac1a239011ce8387242547966438c4744f8a25efe1bfa6",
|
||||
"sha256:22fb529ee55983ef3c46b11771d3d84f58ce3216deaceb303b18b290eae66608",
|
||||
"sha256:28a237c01b9d5fa2120ee4606e1ca3a989f55b67698458b76a793c1a48671887",
|
||||
"sha256:2a71354c2da18cd117a88d1bc983bf6287abc465ccd5357360091c1d4726161b",
|
||||
"sha256:2db844b990c2160d42dc6ed1765e9de68a319bd67d1bcbbf8f71a7a6be013c40",
|
||||
"sha256:329621ab7581d6fc806b60d1fe88e48a4f568d44a2406954cf6fdb3344e83d2e",
|
||||
"sha256:3313708237df9efd58092915fef3ee222585c02a12051f06616566db0999559f",
|
||||
"sha256:350986411597c90cc257010fcb451261716d0814ca1cb6882f2cc303a17cf61e",
|
||||
"sha256:3938b80bb38a5370d34659e4e16f2be7750854ab4f2b524e7a534b610b93ae91",
|
||||
"sha256:393a9cf2f2c0f79e1575e9a1ff0cb460337b2bdc7f56b068237d4f1a269e03e6",
|
||||
"sha256:3a42be69b137494f26219674d38ca3801e5db2fe60de02c332f19d6597729544",
|
||||
"sha256:3ac5f8f9a1ed671b126975b28bd0696d033c701fd24ebaf18c2a4af38b7db183",
|
||||
"sha256:3ba113b254cb22406199a72592fa14acb9713128cedaa6156153fe8cf66d7417",
|
||||
"sha256:3d4f8f746521aa6f9d1d1bfacbf6087449cb8946139f5ce50e7a070db96554b3",
|
||||
"sha256:3f6fe03aaceca011d151b462084b37f3001f6279cce04537319876e432d586f1",
|
||||
"sha256:41637188b8d14fb75140ca4d569c7c2de10f025ba89960753766b1c253dc6f79",
|
||||
"sha256:448cc450e540da6650100a462dc5acb3e6972de82d102ee94015d3ab47071dbc",
|
||||
"sha256:46d809ff965a285b3890ed65aaf84f75876cb5d568ccbe6f1f58a1a3a44c4b94",
|
||||
"sha256:46df6d133d089b22a5ff53e542851631a09e69a022f3822c62f9dfe0e558c8b9",
|
||||
"sha256:487aeea9781651a941f32107720691e2cdc6ec51ef4a97a9e2ea793b0ef66a80",
|
||||
"sha256:4c238565d5a1321167cbb7b5e818bd07fef56b46896c0a05a9a3c5db0d3b5bcd",
|
||||
"sha256:4ccf312f7ed82581713ffa502420dc8715ee908325a8ba3e53490c5da578720c",
|
||||
"sha256:4e33686827f82549c100cdb70b9a301e1dd77593770dd7a2f9c9775b39d779cc",
|
||||
"sha256:4fae096efc700ae2d5269dae1d008b6c841f9044f1090c475ab4994ba19d5cfc",
|
||||
"sha256:5086b1504e715647fa608d6dc6b801a273d14e68a52d1c48a55feab125f1a065",
|
||||
"sha256:5494d247cd6df5fc4c4115cdb4254ba1530c0d4b67e9a228b401938eed8565ac",
|
||||
"sha256:5707da68092852ed75e01c302eaa161b8dbf04c06cb78e574c5142ecdf26f358",
|
||||
"sha256:59985cfa7f655c97ad29f6c6f10e2d052a90809eda79fe9504107cdaccee3448",
|
||||
"sha256:5ff9c64d53c551cba31617510600907b85c706457138e2b4770bb8cea722c0d3",
|
||||
"sha256:6050dbc9dcaeb389099c7ab153296237e9f0ee89705b96a7d5b936c96a0fab04",
|
||||
"sha256:6097021b35ff52e86a4a476acea7c0efd71a9483b9cfc6b15b0d4c1f8e2f1fd7",
|
||||
"sha256:63950d0ba16d1f89a8edbdebf2b82bcc5e724e813f0c0f4c7f109b823fecbf87",
|
||||
"sha256:6771d2930dea96be6f6dc60bdd21db77b4dea2f8bd09752c013b1f6241c7e112",
|
||||
"sha256:6812683714b3f8c7048e7c441b20cc4ef6213a2335486d555e4217f0726c7b55",
|
||||
"sha256:698e05518bf193de0b76388f2136889840f28effb34768ca4a763523e6ee9245",
|
||||
"sha256:69ab30ac5fb24f943416884b7cd88515e5cdfe875d25cfd4edea04ba6a6f8c81",
|
||||
"sha256:752b752c0d40497a6702533a57c2739defe94765b563eddc7a929d8d3e1a471b",
|
||||
"sha256:76899f5b699e32c49475231e90ea8cea61321a4462dc44bd10c1805ce45b0376",
|
||||
"sha256:778d7d81d4233f26bf53ea27fed6f2af10216d2e81706c5e952461c454a264dc",
|
||||
"sha256:7a9900ef63b202746210f2ba86706accc4d86c6a47ad0cd0a229b8800230ec94",
|
||||
"sha256:7a9fe7e7471c40043ca9d8a85d2966f6375b0d42735a42266f9eb4e3afdcc33e",
|
||||
"sha256:8667e1d9faae729d39c409c649f6b83244eb88690e362f78461ec20d52afc7a1",
|
||||
"sha256:89c96e1ba5ae1ddec0655589e343c09826d4ea4118deb4b4173f6f971999dee8",
|
||||
"sha256:8c162c5e145a2975e45e77b60e2b3340252df49507153def80064a10696b513a",
|
||||
"sha256:8c49d65d207338b8dfe6cd0b788c7f86d5c61dd7c027c155ba3db4ca0a4ad948",
|
||||
"sha256:8d9612112fa5311a38625c6f61d28ea2ae48a12d739ffef12bbfc42c0c2a2dde",
|
||||
"sha256:8f0c58dd702928aedc0072fcb43dd235f84cebccc689064cdf2f935b6a154d4b",
|
||||
"sha256:8fb2eec707276804abe731e918fffc127fb43ba7497741bf274ef5bdb5c077e8",
|
||||
"sha256:900135e33d9260ab929cd8f5a88e4a2458eef7b34a87aa386d6c39d05923bb9d",
|
||||
"sha256:911d5f61fcd56ac7c6bfc4568fdf2c4c1c3a9e2a8976a3274166a03e35249e28",
|
||||
"sha256:917ad33e2b4968e28bf780033a188736a99946a9d91df1be8eae60256c82ace6",
|
||||
"sha256:91e92044bcb6e7d4860336cac197f487ecbc433bea443e2e0e90a8dc692ca16c",
|
||||
"sha256:964cc82002c91b3acc25b9b77da7981885e273a6cd601c1f15e732aa1ac744be",
|
||||
"sha256:97db0360c94674657217b0e0bd83cae85a820ae91030729869c3f955a7b056f1",
|
||||
"sha256:98165faf913868f66afaa2f3293ad69bef36308e8a75480a0567bc35738c4cc3",
|
||||
"sha256:9b5aec9e01f4c9b528d96320fdf560398ecd6f6aa65476390e5b8dc7e2cb701c",
|
||||
"sha256:9e576b4f9670412c7cac8eefa1a969aecb3d1879bf6daca2560244cf795563ca",
|
||||
"sha256:a2c2fa250e7acef8712d842c1e5ac765da7f12084813867f168ea83b4bd5cda2",
|
||||
"sha256:a341e0200a34603ee81deb124de597e6c723d67171ca058966945bd68d77a6d1",
|
||||
"sha256:a5836dfc928e197f53126d1ee89f5b73f194218a28c3aaf7ec0d8e1ba886f147",
|
||||
"sha256:a608474e07b28adf954850a2be7d1ee25572d6dc90d1323bc0932dc9b807ffdd",
|
||||
"sha256:a8e700f49d6174c06637cab489cc593c13223181e2c00bbec8de782140743144",
|
||||
"sha256:ab7cd8a404a086b30c671c78377a0806dc113e676a4af80c85323f0bb2f85300",
|
||||
"sha256:ade4d7954a18560529aa1bc0930b0dfff5a3fe5e6772a3a0810307c09d61af31",
|
||||
"sha256:b30fc99e69971b91ae80c37b87c4e66dcb82d968bfe3a59b37bcae0f7a58bd49",
|
||||
"sha256:b9c1bab76af86908aa1115b7b8fa5e65ebf0d0adc7d9fc8e27a3bc8a14dc6202",
|
||||
"sha256:bc60251893baa93f3c4b292764b17b8ca6fbb50972de9f7fcc248634aea2fade",
|
||||
"sha256:bdbb7da34e9445915fd4ddcd53a360655d3f472b4bfa52b662b2f938573d53ab",
|
||||
"sha256:c704216ffd31b5eec26cd998117fd24d33c65f8b5930faf200257670bd3ff1f3",
|
||||
"sha256:c709f4b02492c28936bca9e2f99a554f7fb527bdd76f2b17ffb4fd0b0107d807",
|
||||
"sha256:c817ec812a21a7d3fa7b772aa56704a4cf4d8f79453600c4e30b2a4a163a1f85",
|
||||
"sha256:c9424f15818595de1cf6f7f9ec4e01848d6ff8335aaaed3aee4836c2a4776312",
|
||||
"sha256:cb769206eeb69df28de4eb4a38480ff07dc7ef236a8ed6784908d45db7b1946b",
|
||||
"sha256:cc5cfce70b40e4ccbcadbeac3fc10c14db18766822c9d671f6d9cbc7c6dd031c",
|
||||
"sha256:d21b6f1710125cccc5d5c7be4fa9d06007538ab4af1fab5c0028e286fb99447a",
|
||||
"sha256:d30acdacb94cd6da75569ce9b5c22bb04dec35e4411e4b483f2d683d0d4c8cd3",
|
||||
"sha256:d400e56a203fcfd9275b11e08114b5dc17925ad58fff84a9ac4ec37a6a611ad4",
|
||||
"sha256:d7e24f5729d4d09e50a9594633a600e51f1423aa685706ca87b054cb2f220df3",
|
||||
"sha256:d9b7b622bc7e47854ea660683d4722e6057a5a07176f38c54ce5da8ea7b53741",
|
||||
"sha256:dbe04ea4a9aa1d4210bf1e8968f94695acad72ef02626c1cfdef9a4539560788",
|
||||
"sha256:dc5abb94be9f670e06622531042c5d3f1c1fd338c872ed2b2c9a164bdbfdf75a",
|
||||
"sha256:def88ea5e95c202f89a60809c0ba13919f9e91ac6e94d8fa0cfcc6e13169fbd8",
|
||||
"sha256:e3c18a6e8ce823ac113f3e69c220bb5137ad60eff4ffe01c56d42dc8fc0bffa0",
|
||||
"sha256:ed3f4f39d219b7f304aec0b28a95c273111b69ffcb5b87d61b1fb3601dd964f9",
|
||||
"sha256:edbb829d5b7864385fbdeca8c0b221249a9fcc4b852d0d4f92db60f93251dc9a",
|
||||
"sha256:ee9f9736fcaec21d182fb2fc359c8ce4c92e606b0f2b60c8c02aac8e2b345666"
|
||||
"sha256:019ae21de930d6077efa1eac746de4df5234e7c6c11ab10080c0935fc5abbecf",
|
||||
"sha256:02688fff6d256afdd57da5359144ddab8e054b2ba98ddcf147fe191bdf996e88",
|
||||
"sha256:0274b87df89d1dda8dce77cf05a9dfab7bd30045a09e0d9435ec8be622e374e6",
|
||||
"sha256:0323e8dbeec4d63c27111796baa7e8a89b391c32d90e67d78f9404d0c8edeab4",
|
||||
"sha256:053edbb52fe8b8a1a6698c4fee39590c9e44a602ace807291eb87e3b17f85f48",
|
||||
"sha256:059027f5dd2aafb916301f46a619c7fe03ff5761cdb2d091cf80bf6dbc24bc29",
|
||||
"sha256:05f11a4be4f668974238cff21208fbd9f629cab8a68b444b7d4a4cfd8081b1d6",
|
||||
"sha256:0ab71cc5ea86f6685a7b2235edad65f1f2a4b6341109af259d758973d96eece5",
|
||||
"sha256:0b439f4fb0b615bc0443cc83eaf5835bd480f680c69ed1be963bdb401b8159f8",
|
||||
"sha256:0ec50d24a12e50857e94ac9035d3c06fd0827bb477b9ebcd83a2a49dd89e5e23",
|
||||
"sha256:131fc50d52a52acc367ea8bccb028447b734243d00ba1cfc7d9ff8d0dc37fa38",
|
||||
"sha256:17b5f1d1a4a5ac536283298c98cafc5632ae3897c8601fb2ec8babc6f47a1be9",
|
||||
"sha256:183b8da9b870ad171a11a629c43e0587a228aea9d595a969231d59bf530b6c77",
|
||||
"sha256:18888d50813b9df9b8dc8c1506ec40c783db25f130a6101eb89896b27076f751",
|
||||
"sha256:25b88277832eb558305c3bb986ad61f19b5cb5a87aced289bce4a1701a92aa31",
|
||||
"sha256:266cdab48e2242b6c010beb8b7af4164aa87f4ad8d6fbd9f4f531214f8ddb234",
|
||||
"sha256:281bffb09b2e1620db4e99a9df96e38d939c341c7c43cd5191326fbdb4d42275",
|
||||
"sha256:28cd002cf5a499e6e9bd69d992ffd501b8473948f3e97d6e075b774df1901e8e",
|
||||
"sha256:2972c6c6a806e0c788f6ec39510abdb61b3a648fd141a5fa77becd2cc05ff551",
|
||||
"sha256:2b4027b370cc46c4802ba32a979729209c0407d548723e809f19a50a9df27405",
|
||||
"sha256:318c924e218be754427ce6bb4c630d9dcb5478eb00a8a3f8a0972086adc763b1",
|
||||
"sha256:380accae56f8c9df99f34bc7e79d286fee37c3dd06b362c394b08ea96371b7c5",
|
||||
"sha256:3c7784f9936292c9d3f92fc772d874edc071a16cd883ea0d997e5c4318f6362c",
|
||||
"sha256:3ebd85fd6253abe89f852fc008294d490eb7a5f66913703148b8d263b048cc90",
|
||||
"sha256:4126c8fe9d817ac3ab223ee5db41a09d0fa82dbd6bb59d207b6f7313d733f19b",
|
||||
"sha256:4155f0ab246b6892110960f25989ab91073cd708b974f4732dca4d219a8be3e1",
|
||||
"sha256:41f16267d8e6d916e06a6a1a0e151f643a6bab1277945a4bd494f359d4185dd2",
|
||||
"sha256:4522f5d662d3ee55a072fad18e2af5dae480658d4e23b04b455c4b7542ce4327",
|
||||
"sha256:46c900c807b0614c454ba89271ec6f59212403c54dc68ea493ab1ece2c510618",
|
||||
"sha256:48291b25a904243f37c9aabbfed3eaba466c9a993f5f5946fe647163b7face07",
|
||||
"sha256:5038a5e9e106087c117f0a7d6fd9d8a382b228da24bbd085b9f2b5d54ab11c3a",
|
||||
"sha256:594a26bcf0cb720c16ac6db3fd4b3f411be756f9da7682f2f629089ff15aef18",
|
||||
"sha256:59706135d3107939effe9f9263bd78c507f4abd7bfb96acc5a7f4176aa0a90d2",
|
||||
"sha256:5a327d7581696c7a392a8f85cce7e54fa1303f5b79b3b2983abaab309b56cfd6",
|
||||
"sha256:5eca8a45d38c916783c44e5da06a367b77234efa51d84dda8804654b99efecc9",
|
||||
"sha256:5fa85f6789178ede5333568cbee5bac5fa9718d5f02406b65545e83368fa8fe9",
|
||||
"sha256:65097e45ef7a942a9b92999b81d2e91fe80cbd0616215e625af39d2166692018",
|
||||
"sha256:65cc9938cb9bd8862fc220e0719fd7f9c291d788f0a62bb8840820c46fa5a4d0",
|
||||
"sha256:6a4c3607e2a0e66337d8ddf95ca7efe9b30ebf944119a4fb86503ea66f777263",
|
||||
"sha256:72f11a136f148eb1218e7d1492749b8b5594302010db0cebd47423c4ac8c79ee",
|
||||
"sha256:78b5a71de59e30c697a64c69fc48b032bb99c43b7437091b808a9ba20bb0235c",
|
||||
"sha256:7b212edc9bf9d0c25cc3117483289b9e1a49a1ed134a02635baa987e9f0d89db",
|
||||
"sha256:7e0f7045c420abdea249a28384baa846b87bad5c9f42af1957dc50c6e337fa1a",
|
||||
"sha256:7e83cfec424f546dc3f0cc71896f8cc384a711f4116bc1abb0598302a9af3240",
|
||||
"sha256:80c55bcc31d21bd07f7d1589e11f2ac1faf3359cf9f93026a1944ee76a40f954",
|
||||
"sha256:863740d7f45adfd29b95658a680b16113721eaa89857c67e7e9573c61e87bbd8",
|
||||
"sha256:88484b8c3f71dc9205d0d36da541e2cdcf4bc74474a2ee8d99c2e6411b659b89",
|
||||
"sha256:8a08810e0bcc606d10cf1c5389c96fc92362244c0cf761358c495c2eb29df3dc",
|
||||
"sha256:8c0637ae4fcb54d5c7fc9af24d348003b6f9dbaf7a06bf13f769d7b85903af39",
|
||||
"sha256:8e9e3409338a42e3d4c30c224fdb678364542c77994f089fd6cc8131969eff48",
|
||||
"sha256:902ea10ba85e014dc5d23a7bbb3ab70722349561e73783dd71571359e8867244",
|
||||
"sha256:9533db74a2685169380db3db3ab59643453e7c486fffa9bf3ab60b73c4e174be",
|
||||
"sha256:97f02ff49d1fa21308207a7743bec4fdd7aa90e8dd091539da660fc51e624c4d",
|
||||
"sha256:9ea9a2a154dc7d8658930fa87cda0e6094235b5e130f037d9894eaf8722119a5",
|
||||
"sha256:a0440d847b2c9986e4d27e8a59164714e5198530c69a5f9fb2e4620f9136d653",
|
||||
"sha256:a6d39a27b542a781d691827b955d685d496fb6cccfc6eecc336a78b399032062",
|
||||
"sha256:a7f4d3c478b1fcf412bf6c82914b02fed33ab359120df9172dda7bc855227461",
|
||||
"sha256:ad297807bbdffce61b04e5e0c22f3c5d9e1905c1ee186f1f6d029f83bf0f18b8",
|
||||
"sha256:add6778bb51efb80174937543754d2dfa0f4e504e7302d97896006a642c14f95",
|
||||
"sha256:ae075ebf7bb5f48b3bd2fc9cd53346e4ff43e2515a4f822914bbc62a3cbd6e7e",
|
||||
"sha256:b26fb439a7fbb522af63bbd781fbf51ec0c0659134a93f5bc8e9e68641df811e",
|
||||
"sha256:b2bac59721d246939b21274229b9923aeae3db97b6118da739c658c17e110dd6",
|
||||
"sha256:b314ad1f0667715e8d1b6197d5336ab579b13e801172721d62331bd40034a30c",
|
||||
"sha256:b7317035875bd7c4705e2566848b2043b78e18f2f5675ea651f9f7805b5589eb",
|
||||
"sha256:b8e936e620e5f336a207e08c0da9dace5d4dbcc8e64743ab1acaa77a64bbf060",
|
||||
"sha256:b906da4e9a7ba4ec33ed2f7238343866932c1a6f84944c804252b2922708d0ee",
|
||||
"sha256:ba690e4e33c360fcf0b8411ca90f8b9cc595e8deddd6a25a9a75a725b698cd6a",
|
||||
"sha256:bb14da3d63da994c34cfa47cde469df8013ddf5f575455a22530c8c4a0ed8616",
|
||||
"sha256:bbc2e1632f4a61fa171ddab3bc8368fb8475e7ce68733ca92fec862fdd8e0f60",
|
||||
"sha256:bbdd3c896db09993b7879cd35e56da6ed8918d161d6e80f9d9c40d78d34e4784",
|
||||
"sha256:bcaaa8e542cb7e1962d0a58ce6a25f6b4b6ca2e5ce743155fc1f6eb2fea52574",
|
||||
"sha256:bee682ab1005aff597946234e47c95fcf0f44d2b1f38075f0aba26bbc4e7545a",
|
||||
"sha256:bfec6543d60c57e7543d9cbccdd5dfcf562f2c05cd6b814df68108a20794e254",
|
||||
"sha256:c2e50baf7be8831524a87beec6c1873539519a1948f907dc3d4b9be27ebacb80",
|
||||
"sha256:c6c79a6138be017d85f3bab1df735669b669a38f9b3ff646a1f179afbacb7b63",
|
||||
"sha256:c702fb7c8bfd87c9ce9c8bddfc9a5796a492bab35a52b1693adee413721e32f2",
|
||||
"sha256:c9ba1725826f6571a6e4c1561bb1613711f0058b91927a147dc42c637ba087d9",
|
||||
"sha256:cf205ac52cb6b45745c0a4891cdb6e709c10ad5b034aa736aff561fc4ce9828c",
|
||||
"sha256:d0d03fc67499ee90feedfa2add4aaa1c091a7bf333535d847b10fffe390e58fe",
|
||||
"sha256:d118d63f08fd6ac285cb8166e96c992a6ed0e7a1644e8790c39070b18779e688",
|
||||
"sha256:d24c09f397c3ce55f20e0250da7ba5b0e5249cb5d21465e71ec15154a3a7e8e0",
|
||||
"sha256:d41735c7a646dae8612e0552dfc53f45807eeb54364dfb1f0a65ac274bc56b3a",
|
||||
"sha256:dd1696d91f2a37cece9bd22e507e7be7c37c59ecc61fd15f0d0f31e3b6888957",
|
||||
"sha256:dfcad9c63a893c95ba1149481b9680ce68dd71211f08df0073ee62700790bc97",
|
||||
"sha256:e384782608837d9aaf123e413679883091744664a2cd76f0ad0e0a1f12facc57",
|
||||
"sha256:e5ea0abea338c617b753082f36f64c70ade853d88e91ab5732b301ae8ed16e3f",
|
||||
"sha256:e6ff81c570413bcc35f1c16850eb66e2493a3259e68efe8672376533d2c82d38",
|
||||
"sha256:e88951ad2831880405f3f055ab12a6aa72696c20a2815128eeccdc3bf914cd78",
|
||||
"sha256:e98e16b6ce531b12100c01daac922e8ec5b991832a5f58003f13b7d45ea82dc0",
|
||||
"sha256:eb0fd32e8e433797499571447d9f975b4744be79c0a3339413868d79517231ed",
|
||||
"sha256:ee74a73e1f9e16b71f67329e99bb58aa4af9a2c3c4b3a5db9f26e92e7c39e161",
|
||||
"sha256:f15ec5f825c283a5aa427d78759ab8f84e7b5441d15cfff476b548bce3764666",
|
||||
"sha256:f296c7fe928ce0e29e313f85c43a5ab80542e096e1163c2605b8cc18aa2aff2b",
|
||||
"sha256:f32df1b19f773bb41382e8b215955d248c9766e3d6ff5a1dd89709e7d96e4685",
|
||||
"sha256:f3ed67279a4b317a808ac743d3a915f74187530c5f3d9c859e5d04d475b8c174",
|
||||
"sha256:f5b972ca514898fb7131671c425a62ca38fdae2a8d6296e4b605ec8202349f8c",
|
||||
"sha256:f961086c0dbba6c00cbd5c5b5646247efd0d0a4044444bfaa9efc7a6ba5e96a5",
|
||||
"sha256:f9bd7d7a449667d6f17edd9045ec82a4ed2767afb91743d3d0b18c376a56dfe2",
|
||||
"sha256:fbac4c8ffadb685189efa92fafdb2f5392e9cbd262eae3818bcdb1bd19acaaf2",
|
||||
"sha256:fc43c8276d0a7c7b76f31d4f3f80f9eb820673628f1411770a70029c1d5f6a75",
|
||||
"sha256:fcfded324f0710632e22050a2fd7b56b1cbcb2d21001630bcc26d536f54bffec",
|
||||
"sha256:ff435abdcbfdf4a070f488830cd53aef77cf8649d0fd8ed76bf27d9566e80e78"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==0.20.5"
|
||||
"version": "==0.20.7"
|
||||
},
|
||||
"lxml": {
|
||||
"hashes": [
|
||||
@ -1195,10 +1195,10 @@
|
||||
},
|
||||
"python-levenshtein": {
|
||||
"hashes": [
|
||||
"sha256:114dfa996d5aa91d400dd816210a62523be8c598962aebbf7b086c7fd26fb72d",
|
||||
"sha256:f55310dc28bb891428d751299688e0671fa3e0405ddd7561884747d6b1be2e7d"
|
||||
"sha256:88a58b95e3340a918489dac0c78f731323c0a4d8f5564f839ffea80155574e77",
|
||||
"sha256:9228af5523f797f0798f045dc4a95ed1f46df72bc2186e52b530a33998a51b37"
|
||||
],
|
||||
"version": "==0.20.5"
|
||||
"version": "==0.20.7"
|
||||
},
|
||||
"python-magic": {
|
||||
"hashes": [
|
||||
@ -1654,11 +1654,11 @@
|
||||
},
|
||||
"setuptools": {
|
||||
"hashes": [
|
||||
"sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012",
|
||||
"sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"
|
||||
"sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17",
|
||||
"sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==65.4.1"
|
||||
"version": "==65.5.0"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
@ -1736,11 +1736,11 @@
|
||||
},
|
||||
"tzdata": {
|
||||
"hashes": [
|
||||
"sha256:74da81ecf2b3887c94e53fc1d466d4362aaf8b26fc87cda18f22004544694583",
|
||||
"sha256:ada9133fbd561e6ec3d1674d3fba50251636e918aa97bd59d63735bef5a513bb"
|
||||
"sha256:323161b22b7802fdc78f20ca5f6073639c64f1a7227c40cd3e19fd1d0ce6650a",
|
||||
"sha256:e15b2b3005e2546108af42a0eb4ccab4d9e225e2dfbf4f77aad50c70a4b1f3ab"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2022.4"
|
||||
"version": "==2022.5"
|
||||
},
|
||||
"tzlocal": {
|
||||
"hashes": [
|
||||
@ -2137,7 +2137,7 @@
|
||||
"sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845",
|
||||
"sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"
|
||||
],
|
||||
"markers": "python_full_version >= '3.6.0'",
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2.1.1"
|
||||
},
|
||||
"click": {
|
||||
@ -2259,11 +2259,11 @@
|
||||
},
|
||||
"faker": {
|
||||
"hashes": [
|
||||
"sha256:245fc7d23470dc57164bd9a59b7b1126e16289ffcf813d88a6c8e9b8a37ea3fb",
|
||||
"sha256:84c83f0ac1a2c8ecabd784c501aa0ef1d082d4aee52c3d797d586081c166434c"
|
||||
"sha256:096c15e136adb365db24d8c3964fe26bfc68fe060c9385071a339f8c14e09c8a",
|
||||
"sha256:a741b77f484215c3aab2604100669657189548f440fcb2ed0f8b7ee21c385629"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==15.0.0"
|
||||
"version": "==15.1.1"
|
||||
},
|
||||
"filelock": {
|
||||
"hashes": [
|
||||
@ -2608,11 +2608,11 @@
|
||||
},
|
||||
"setuptools": {
|
||||
"hashes": [
|
||||
"sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012",
|
||||
"sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"
|
||||
"sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17",
|
||||
"sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==65.4.1"
|
||||
"version": "==65.5.0"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
@ -2631,11 +2631,11 @@
|
||||
},
|
||||
"sphinx": {
|
||||
"hashes": [
|
||||
"sha256:5b10cb1022dac8c035f75767799c39217a05fc0fe2d6fe5597560d38e44f0363",
|
||||
"sha256:7abf6fabd7b58d0727b7317d5e2650ef68765bbe0ccb63c8795fa8683477eaa2"
|
||||
"sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d",
|
||||
"sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.2.3"
|
||||
"version": "==5.3.0"
|
||||
},
|
||||
"sphinx-autobuild": {
|
||||
"hashes": [
|
||||
@ -2739,7 +2739,7 @@
|
||||
"sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e",
|
||||
"sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b"
|
||||
],
|
||||
"markers": "python_version > '2.7'",
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==6.2"
|
||||
},
|
||||
"tox": {
|
||||
|
||||
@ -249,16 +249,22 @@ class RasterisedDocumentParser(DocumentParser):
|
||||
|
||||
if mime_type == "application/pdf":
|
||||
text_original = self.extract_text(None, document_path)
|
||||
original_has_text = text_original and len(text_original) > 50
|
||||
original_has_text = text_original is not None and len(text_original) > 50
|
||||
else:
|
||||
text_original = None
|
||||
original_has_text = False
|
||||
|
||||
# If the original has text, and the user doesn't want an archive,
|
||||
# we're done here
|
||||
if settings.OCR_MODE == "skip_noarchive" and original_has_text:
|
||||
self.log("debug", "Document has text, skipping OCRmyPDF entirely.")
|
||||
self.text = text_original
|
||||
return
|
||||
|
||||
# Either no text was in the original or there should be an archive
|
||||
# file created, so OCR the file and create an archive with any
|
||||
# test located via OCR
|
||||
|
||||
import ocrmypdf
|
||||
from ocrmypdf import InputFileError, EncryptedPdfError
|
||||
|
||||
@ -276,8 +282,6 @@ class RasterisedDocumentParser(DocumentParser):
|
||||
self.log("debug", f"Calling OCRmyPDF with args: {args}")
|
||||
ocrmypdf.ocr(**args)
|
||||
|
||||
# Only create archive file if archiving isn't being skipped
|
||||
if settings.OCR_MODE != "skip_noarchive":
|
||||
self.archive_path = archive_path
|
||||
|
||||
self.text = self.extract_text(sidecar_file, archive_path)
|
||||
|
||||
@ -341,6 +341,17 @@ class TestParser(DirectoriesMixin, TestCase):
|
||||
|
||||
@override_settings(OCR_PAGES=2, OCR_MODE="redo")
|
||||
def test_multi_page_analog_pages_redo(self):
|
||||
"""
|
||||
GIVEN:
|
||||
- File with text contained in images but no text layer
|
||||
- OCR of only pages 1 and 2 requested
|
||||
- OCR mode set to redo
|
||||
WHEN:
|
||||
- Document is parsed
|
||||
THEN:
|
||||
- Text of page 1 and 2 extracted
|
||||
- An archive file is created
|
||||
"""
|
||||
parser = RasterisedDocumentParser(None)
|
||||
parser.parse(
|
||||
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"),
|
||||
@ -352,6 +363,17 @@ class TestParser(DirectoriesMixin, TestCase):
|
||||
|
||||
@override_settings(OCR_PAGES=1, OCR_MODE="force")
|
||||
def test_multi_page_analog_pages_force(self):
|
||||
"""
|
||||
GIVEN:
|
||||
- File with text contained in images but no text layer
|
||||
- OCR of only page 1 requested
|
||||
- OCR mode set to force
|
||||
WHEN:
|
||||
- Document is parsed
|
||||
THEN:
|
||||
- Only text of page 1 is extracted
|
||||
- An archive file is created
|
||||
"""
|
||||
parser = RasterisedDocumentParser(None)
|
||||
parser.parse(
|
||||
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"),
|
||||
@ -395,7 +417,7 @@ class TestParser(DirectoriesMixin, TestCase):
|
||||
- Document is parsed
|
||||
THEN:
|
||||
- Text from images is extracted
|
||||
- No archive file is created
|
||||
- An archive file is created with the OCRd text
|
||||
"""
|
||||
parser = RasterisedDocumentParser(None)
|
||||
parser.parse(
|
||||
@ -408,15 +430,26 @@ class TestParser(DirectoriesMixin, TestCase):
|
||||
["page 1", "page 2", "page 3"],
|
||||
)
|
||||
|
||||
self.assertIsNone(parser.archive_path)
|
||||
self.assertIsNotNone(parser.archive_path)
|
||||
|
||||
@override_settings(OCR_MODE="skip")
|
||||
def test_multi_page_mixed(self):
|
||||
"""
|
||||
GIVEN:
|
||||
- File with some text contained in images and some in text layer
|
||||
- OCR mode set to skip
|
||||
WHEN:
|
||||
- Document is parsed
|
||||
THEN:
|
||||
- Text from images is extracted
|
||||
- An archive file is created with the OCRd text and the original text
|
||||
"""
|
||||
parser = RasterisedDocumentParser(None)
|
||||
parser.parse(
|
||||
os.path.join(self.SAMPLE_FILES, "multi-page-mixed.pdf"),
|
||||
"application/pdf",
|
||||
)
|
||||
self.assertIsNotNone(parser.archive_path)
|
||||
self.assertTrue(os.path.isfile(parser.archive_path))
|
||||
self.assertContainsStrings(
|
||||
parser.get_text().lower(),
|
||||
@ -438,7 +471,7 @@ class TestParser(DirectoriesMixin, TestCase):
|
||||
- Document is parsed
|
||||
THEN:
|
||||
- Text from images is extracted
|
||||
- No archive file is created
|
||||
- No archive file is created as original file contains text
|
||||
"""
|
||||
parser = RasterisedDocumentParser(None)
|
||||
parser.parse(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user