diff --git a/.github/scripts/cleanup-tags.py b/.github/scripts/cleanup-tags.py index 904f44346..b89bd8ae0 100644 --- a/.github/scripts/cleanup-tags.py +++ b/.github/scripts/cleanup-tags.py @@ -8,6 +8,7 @@ from argparse import ArgumentParser from typing import Dict from typing import Final from typing import List +from typing import Optional from common import get_log_level from github import ContainerPackage @@ -26,7 +27,7 @@ class DockerManifest2: def __init__(self, data: Dict) -> None: self._data = data - # This is the sha256: digest string. Corresponds to Github API name + # This is the sha256: digest string. Corresponds to GitHub API name # if the package is an untagged package self.digest = self._data["digest"] platform_data_os = self._data["platform"]["os"] @@ -38,6 +39,269 @@ class DockerManifest2: self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}" +class RegistryTagsCleaner: + """ + This is the base class for the image registry cleaning. Given a package + name, it will keep all images which are tagged and all untagged images + referred to by a manifest. This results in only images which have been untagged + and cannot be referenced except by their SHA in being removed. None of these + images should be referenced, so it is fine to delete them. + """ + + def __init__( + self, + package_name: str, + repo_owner: str, + repo_name: str, + package_api: GithubContainerRegistryApi, + branch_api: Optional[GithubBranchApi], + ): + self.actually_delete = False + self.package_api = package_api + self.branch_api = branch_api + self.package_name = package_name + self.repo_owner = repo_owner + self.repo_name = repo_name + self.tags_to_delete: List[str] = [] + self.tags_to_keep: List[str] = [] + + # Get the information about all versions of the given package + # These are active, not deleted, the default returned from the API + self.all_package_versions = self.package_api.get_active_package_versions( + self.package_name, + ) + + # Get a mapping from a tag like "1.7.0" or "feature-xyz" to the ContainerPackage + # tagged with it. It makes certain lookups easy + self.all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {} + for pkg in self.all_package_versions: + for tag in pkg.tags: + self.all_pkgs_tags_to_version[tag] = pkg + logger.info( + f"Located {len(self.all_package_versions)} versions of package {self.package_name}", + ) + + self.decide_what_tags_to_keep() + + def clean(self): + """ + This method will delete image versions, based on the selected tags to delete + """ + for tag_to_delete in self.tags_to_delete: + package_version_info = self.all_pkgs_tags_to_version[tag_to_delete] + + if self.actually_delete: + logger.info( + f"Deleting {tag_to_delete} (id {package_version_info.id})", + ) + self.package_api.delete_package_version( + package_version_info, + ) + + else: + logger.info( + f"Would delete {tag_to_delete} (id {package_version_info.id})", + ) + else: + logger.info("No tags to delete") + + def clean_untagged(self, is_manifest_image: bool): + """ + This method will delete untagged images, that is those which are not named. It + handles if the image tag is actually a manifest, which points to images that look otherwise + untagged. + """ + + def _clean_untagged_manifest(): + """ + + Handles the deletion of untagged images, but where the package is a manifest, ie a multi + arch image, which means some "untagged" images need to exist still. + + Ok, bear with me, these are annoying. + + Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest. + These images are untagged, but pointed to, and so should not be removed (or every pull fails). + + So for each image getting kept, parse the manifest to find the digest(s) it points to. Then + remove those from the list of untagged images. The final result is the untagged, not pointed to + version which should be safe to remove. + + Example: + Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to + amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690 + armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3 + arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b + each of which appears as untagged image, but isn't really. + + So from the list of untagged packages, remove those digests. Once all tags which + are being kept are checked, the remaining untagged packages are actually untagged + with no referrals in a manifest to them. + """ + # Simplify the untagged data, mapping name (which is a digest) to the version + # At the moment, these are the images which APPEAR untagged. + untagged_versions = {} + for x in self.all_package_versions: + if x.untagged: + untagged_versions[x.name] = x + + skips = 0 + + # Parse manifests to locate digests pointed to + for tag in sorted(self.tags_to_keep): + full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}" + logger.info(f"Checking manifest for {full_name}") + try: + proc = subprocess.run( + [ + shutil.which("docker"), + "manifest", + "inspect", + full_name, + ], + capture_output=True, + ) + + manifest_list = json.loads(proc.stdout) + for manifest_data in manifest_list["manifests"]: + manifest = DockerManifest2(manifest_data) + + if manifest.digest in untagged_versions: + logger.info( + f"Skipping deletion of {manifest.digest}," + f" referred to by {full_name}" + f" for {manifest.platform}", + ) + del untagged_versions[manifest.digest] + skips += 1 + + except Exception as err: + self.actually_delete = False + logger.exception(err) + return + + logger.info( + f"Skipping deletion of {skips} packages referred to by a manifest", + ) + + # Delete the untagged and not pointed at packages + logger.info(f"Deleting untagged packages of {self.package_name}") + for to_delete_name in untagged_versions: + to_delete_version = untagged_versions[to_delete_name] + + if self.actually_delete: + logger.info( + f"Deleting id {to_delete_version.id} named {to_delete_version.name}", + ) + self.package_api.delete_package_version( + to_delete_version, + ) + else: + logger.info( + f"Would delete {to_delete_name} (id {to_delete_version.id})", + ) + + def _clean_untagged_non_manifest(): + """ + If the package is not a multi-arch manifest, images without tags are safe to delete. + """ + + for package in self.all_package_versions: + if package.untagged: + if self.actually_delete: + logger.info( + f"Deleting id {package.id} named {package.name}", + ) + self.package_api.delete_package_version( + package, + ) + else: + logger.info( + f"Would delete {package.name} (id {package.id})", + ) + else: + logger.info( + f"Not deleting tag {package.tags[0]} of package {self.package_name}", + ) + + logger.info("Beginning untagged image cleaning") + + if is_manifest_image: + _clean_untagged_manifest() + else: + _clean_untagged_non_manifest() + + def decide_what_tags_to_keep(self): + """ + This method holds the logic to delete what tags to keep and there fore + what tags to delete. + + By default, any image with at least 1 tag will be kept + """ + # By default, keep anything which is tagged + self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys())) + + +class MainImageTagsCleaner(RegistryTagsCleaner): + def decide_what_tags_to_keep(self): + """ + Overrides the default logic for deciding what images to keep. Images tagged as "feature-" + will be removed, if the corresponding branch no longer exists. + """ + + # Locate the feature branches + feature_branches = {} + for branch in self.branch_api.get_branches( + owner=self.repo_owner, + repo=self.repo_name, + ): + if branch.name.startswith("feature-"): + logger.debug(f"Found feature branch {branch.name}") + feature_branches[branch.name] = branch + + logger.info(f"Located {len(feature_branches)} feature branches") + + # Filter to packages which are tagged with feature-* + packages_tagged_feature: List[ContainerPackage] = [] + for package in self.all_package_versions: + if package.tag_matches("feature-"): + packages_tagged_feature.append(package) + + # Map tags like "feature-xyz" to a ContainerPackage + feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {} + for pkg in packages_tagged_feature: + for tag in pkg.tags: + feature_pkgs_tags_to_versions[tag] = pkg + + logger.info( + f'Located {len(feature_pkgs_tags_to_versions)} versions of package {self.package_name} tagged "feature-"', + ) + + # All the feature tags minus all the feature branches leaves us feature tags + # with no corresponding branch + self.tags_to_delete = list( + set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()), + ) + + # All the tags minus the set of going to be deleted tags leaves us the + # tags which will be kept around + self.tags_to_keep = list( + set(self.all_pkgs_tags_to_version.keys()) - set(self.tags_to_delete), + ) + logger.info( + f"Located {len(self.tags_to_delete)} versions of package {self.package_name} to delete", + ) + + +class LibraryTagsCleaner(RegistryTagsCleaner): + """ + Exists for the off change that someday, the installer library images + will need their own logic + """ + + pass + + def _main(): parser = ArgumentParser( description="Using the GitHub API locate and optionally delete container" @@ -100,190 +364,32 @@ def _main(): # Note: Only relevant to the main application, but simpler to # leave in for all packages with GithubBranchApi(gh_token) as branch_api: - feature_branches = {} - for branch in branch_api.get_branches( - repo=repo, - ): - if branch.name.startswith("feature-"): - logger.debug(f"Found feature branch {branch.name}") - feature_branches[branch.name] = branch - - logger.info(f"Located {len(feature_branches)} feature branches") - - with GithubContainerRegistryApi(gh_token, repo_owner) as container_api: - # Get the information about all versions of the given package - all_package_versions: List[ - ContainerPackage - ] = container_api.get_package_versions(args.package) - - all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {} - for pkg in all_package_versions: - for tag in pkg.tags: - all_pkgs_tags_to_version[tag] = pkg - logger.info( - f"Located {len(all_package_versions)} versions of package {args.package}", - ) - - # Filter to packages which are tagged with feature-* - packages_tagged_feature: List[ContainerPackage] = [] - for package in all_package_versions: - if package.tag_matches("feature-"): - packages_tagged_feature.append(package) - - feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {} - for pkg in packages_tagged_feature: - for tag in pkg.tags: - feature_pkgs_tags_to_versions[tag] = pkg - - logger.info( - f'Located {len(feature_pkgs_tags_to_versions)} versions of package {args.package} tagged "feature-"', - ) - - # All the feature tags minus all the feature branches leaves us feature tags - # with no corresponding branch - tags_to_delete = list( - set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()), - ) - - # All the tags minus the set of going to be deleted tags leaves us the - # tags which will be kept around - tags_to_keep = list( - set(all_pkgs_tags_to_version.keys()) - set(tags_to_delete), - ) - logger.info( - f"Located {len(tags_to_delete)} versions of package {args.package} to delete", - ) - - # Delete certain package versions for which no branch existed - for tag_to_delete in tags_to_delete: - package_version_info = feature_pkgs_tags_to_versions[tag_to_delete] - - if args.delete: - logger.info( - f"Deleting {tag_to_delete} (id {package_version_info.id})", + with GithubContainerRegistryApi(gh_token, repo_owner) as container_api: + if args.package in {"paperless-ngx", "paperless-ngx/builder/cache/app"}: + cleaner = MainImageTagsCleaner( + args.package, + repo_owner, + repo, + container_api, + branch_api, ) - container_api.delete_package_version( - package_version_info, - ) - else: - logger.info( - f"Would delete {tag_to_delete} (id {package_version_info.id})", + cleaner = LibraryTagsCleaner( + args.package, + repo_owner, + repo, + container_api, + None, ) - # Deal with untagged package versions - if args.untagged: + # Set if actually doing a delete vs dry run + cleaner.actually_delete = args.delete - logger.info("Handling untagged image packages") + # Clean images with tags + cleaner.clean() - if not args.is_manifest: - # If the package is not a multi-arch manifest, images without tags are safe to delete. - # They are not referred to by anything. This will leave all with at least 1 tag - - for package in all_package_versions: - if package.untagged: - if args.delete: - logger.info( - f"Deleting id {package.id} named {package.name}", - ) - container_api.delete_package_version( - package, - ) - else: - logger.info( - f"Would delete {package.name} (id {package.id})", - ) - else: - logger.info( - f"Not deleting tag {package.tags[0]} of package {args.package}", - ) - else: - - """ - Ok, bear with me, these are annoying. - - Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest. - These images are untagged, but pointed to, and so should not be removed (or every pull fails). - - So for each image getting kept, parse the manifest to find the digest(s) it points to. Then - remove those from the list of untagged images. The final result is the untagged, not pointed to - version which should be safe to remove. - - Example: - Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to - amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690 - armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3 - arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b - each of which appears as untagged image, but isn't really. - - So from the list of untagged packages, remove those digests. Once all tags which - are being kept are checked, the remaining untagged packages are actually untagged - with no referrals in a manifest to them. - - """ - - # Simplify the untagged data, mapping name (which is a digest) to the version - untagged_versions = {} - for x in all_package_versions: - if x.untagged: - untagged_versions[x.name] = x - - skips = 0 - # Extra security to not delete on an unexpected error - actually_delete = True - - # Parse manifests to locate digests pointed to - for tag in sorted(tags_to_keep): - full_name = f"ghcr.io/{repo_owner}/{args.package}:{tag}" - logger.info(f"Checking manifest for {full_name}") - try: - proc = subprocess.run( - [ - shutil.which("docker"), - "manifest", - "inspect", - full_name, - ], - capture_output=True, - ) - - manifest_list = json.loads(proc.stdout) - for manifest_data in manifest_list["manifests"]: - manifest = DockerManifest2(manifest_data) - - if manifest.digest in untagged_versions: - logger.debug( - f"Skipping deletion of {manifest.digest}, referred to by {full_name} for {manifest.platform}", - ) - del untagged_versions[manifest.digest] - skips += 1 - - except Exception as err: - actually_delete = False - logger.exception(err) - - logger.info( - f"Skipping deletion of {skips} packages referred to by a manifest", - ) - - # Step 3.3 - Delete the untagged and not pointed at packages - logger.info(f"Deleting untagged packages of {args.package}") - for to_delete_name in untagged_versions: - to_delete_version = untagged_versions[to_delete_name] - - if args.delete and actually_delete: - logger.info( - f"Deleting id {to_delete_version.id} named {to_delete_version.name}", - ) - container_api.delete_package_version( - to_delete_version, - ) - else: - logger.info( - f"Would delete {to_delete_name} (id {to_delete_version.id})", - ) - else: - logger.info("Leaving untagged images untouched") + # Clean images which are untagged + cleaner.clean_untagged(args.is_manifest) if __name__ == "__main__": diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 1e130eae0..bccd4fbbd 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -29,6 +29,11 @@ def get_cache_image_tag( def get_log_level(args) -> int: + """ + Returns a logging level, based + :param args: + :return: + """ levels = { "critical": logging.CRITICAL, "error": logging.ERROR, diff --git a/.github/scripts/github.py b/.github/scripts/github.py index 4059f89d0..63f34a1e9 100644 --- a/.github/scripts/github.py +++ b/.github/scripts/github.py @@ -113,14 +113,14 @@ class GithubBranchApi(_GithubApiBase): def __init__(self, token: str) -> None: super().__init__(token) - self._ENDPOINT = "https://api.github.com/repos/{REPO}/branches" + self._ENDPOINT = "https://api.github.com/repos/{OWNER}/{REPO}/branches" - def get_branches(self, repo: str) -> List[GithubBranch]: + def get_branches(self, owner: str, repo: str) -> List[GithubBranch]: """ Returns all current branches of the given repository owned by the given owner or organization. """ - endpoint = self._ENDPOINT.format(REPO=repo) + endpoint = self._ENDPOINT.format(OWNER=owner, REPO=repo) internal_data = self._read_all_pages(endpoint) return [GithubBranch(branch) for branch in internal_data] @@ -189,8 +189,11 @@ class GithubContainerRegistryApi(_GithubApiBase): self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions" # https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}" + self._PACKAGE_VERSION_RESTORE_ENDPOINT = ( + f"{self._PACKAGE_VERSION_DELETE_ENDPOINT}/restore" + ) - def get_package_versions( + def get_active_package_versions( self, package_name: str, ) -> List[ContainerPackage]: @@ -216,6 +219,30 @@ class GithubContainerRegistryApi(_GithubApiBase): return pkgs + def get_deleted_package_versions( + self, + package_name: str, + ) -> List[ContainerPackage]: + package_type: str = "container" + # Need to quote this for slashes in the name + package_name = urllib.parse.quote(package_name, safe="") + + endpoint = ( + self._PACKAGES_VERSIONS_ENDPOINT.format( + ORG=self._owner_or_org, + PACKAGE_TYPE=package_type, + PACKAGE_NAME=package_name, + ) + + "?state=deleted" + ) + + pkgs = [] + + for data in self._read_all_pages(endpoint): + pkgs.append(ContainerPackage(data)) + + return pkgs + def delete_package_version(self, package_data: ContainerPackage): """ Deletes the given package version from the GHCR @@ -225,3 +252,22 @@ class GithubContainerRegistryApi(_GithubApiBase): logger.warning( f"Request to delete {package_data.url} returned HTTP {resp.status_code}", ) + + def restore_package_version( + self, + package_name: str, + package_data: ContainerPackage, + ): + package_type: str = "container" + endpoint = self._PACKAGE_VERSION_RESTORE_ENDPOINT.format( + ORG=self._owner_or_org, + PACKAGE_TYPE=package_type, + PACKAGE_NAME=package_name, + PACKAGE_VERSION_ID=package_data.id, + ) + + resp = self._session.post(endpoint) + if resp.status_code != 204: + logger.warning( + f"Request to delete {endpoint} returned HTTP {resp.status_code}", + ) diff --git a/.github/workflows/cleanup-tags.yml b/.github/workflows/cleanup-tags.yml index 8c2ef87d9..308b7f2ed 100644 --- a/.github/workflows/cleanup-tags.yml +++ b/.github/workflows/cleanup-tags.yml @@ -24,9 +24,26 @@ concurrency: cancel-in-progress: false jobs: - cleanup: - name: Cleanup Image Tags - runs-on: ubuntu-20.04 + cleanup-images: + name: Cleanup Image Tags for ${{ matrix.primary-name }} + runs-on: ubuntu-latest + strategy: + matrix: + include: + - primary-name: "paperless-ngx" + cache-name: "paperless-ngx/builder/cache/app" + + - primary-name: "paperless-ngx/builder/qpdf" + cache-name: "paperless-ngx/builder/cache/qpdf" + + - primary-name: "paperless-ngx/builder/pikepdf" + cache-name: "paperless-ngx/builder/cache/pikepdf" + + - primary-name: "paperless-ngx/builder/jbig2enc" + cache-name: "paperless-ngx/builder/cache/jbig2enc" + + - primary-name: "paperless-ngx/builder/psycopg2" + cache-name: "paperless-ngx/builder/cache/psycopg2" env: # Requires a personal access token with the OAuth scope delete:packages TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }} @@ -50,63 +67,29 @@ jobs: name: Install requests run: | python -m pip install requests - # Clean up primary packages - - - name: Cleanup for package "paperless-ngx" - if: "${{ env.TOKEN != '' }}" - run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx" - - - name: Cleanup for package "qpdf" - if: "${{ env.TOKEN != '' }}" - run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/qpdf" - - - name: Cleanup for package "pikepdf" - if: "${{ env.TOKEN != '' }}" - run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/pikepdf" - - - name: Cleanup for package "jbig2enc" - if: "${{ env.TOKEN != '' }}" - run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/jbig2enc" - - - name: Cleanup for package "psycopg2" - if: "${{ env.TOKEN != '' }}" - run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --is-manifest --delete "paperless-ngx/builder/psycopg2" # - # Clean up registry cache packages + # Clean up primary package # - - name: Cleanup for package "builder/cache/app" + name: Cleanup for package "${{ matrix.primary-name }}" if: "${{ env.TOKEN != '' }}" run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/app" + python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --is-manifest --delete "${{ matrix.primary-name }}" + # + # Clean up registry cache package + # - - name: Cleanup for package "builder/cache/qpdf" + name: Cleanup for package "${{ matrix.cache-name }}" if: "${{ env.TOKEN != '' }}" run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/qpdf" - - - name: Cleanup for package "builder/cache/psycopg2" - if: "${{ env.TOKEN != '' }}" - run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/psycopg2" - - - name: Cleanup for package "builder/cache/jbig2enc" - if: "${{ env.TOKEN != '' }}" - run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/jbig2enc" - - - name: Cleanup for package "builder/cache/pikepdf" - if: "${{ env.TOKEN != '' }}" - run: | - python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --untagged --delete "paperless-ngx/builder/cache/pikepdf" + python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}" + # + # Verify tags which are left still pull + # - name: Check all tags still pull run: | - ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }') - echo "Pulling all tags of ghcr.io/${ghcr_name}" - docker pull --quiet --all-tags ghcr.io/${ghcr_name} + ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }') + echo "Pulling all tags of ${ghcr_name}" + docker pull --quiet --all-tags ${ghcr_name} + docker image list diff --git a/Pipfile b/Pipfile index e3e813c86..0660ef30f 100644 --- a/Pipfile +++ b/Pipfile @@ -71,7 +71,7 @@ pytest-django = "*" pytest-env = "*" pytest-sugar = "*" pytest-xdist = "*" -sphinx = "~=5.1" +sphinx = "~=5.3" sphinx_rtd_theme = "*" tox = "*" black = "*" diff --git a/Pipfile.lock b/Pipfile.lock index e8b13caed..30c622545 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "91a00f3941172f541e45d764727a12280c0be0898ec0e294c6d1283208f9bc92" + "sha256": "68ff2e4e4ebbed482cc7d646337309ffd51140748b83d0b60d22dec9926f2ccb" }, "pipfile-spec": 6, "requires": {}, @@ -218,7 +218,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.1" }, "click": { @@ -234,7 +234,7 @@ "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" ], - "markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'", + "markers": "python_version < '4' and python_full_version >= '3.6.2'", "version": "==0.3.0" }, "click-plugins": { @@ -562,10 +562,10 @@ }, "incremental": { "hashes": [ - "sha256:02f5de5aff48f6b9f665d99d48bfc7ec03b6e3943210de7cfc88856d755d6f57", - "sha256:92014aebc6a20b78a8084cdd5645eeaa7f74b8933f70fa3ada2cfbd1e3b54321" + "sha256:912feeb5e0f7e0188e6f42241d2f450002e11bbc0937c65865045854c24c0bd0", + "sha256:b864a1f30885ee72c5ac2835a761b8fe8aa9c28b9395cacf27286602688d3e51" ], - "version": "==21.3.0" + "version": "==22.10.0" }, "inotify-simple": { "hashes": [ @@ -608,111 +608,111 @@ }, "levenshtein": { "hashes": [ - "sha256:02f3914af0d1d45764e981a4c0637cc747b73e482250d251c4f268c969d54bce", - "sha256:03fe04ce29cf85b62a4cd03a1fff327ba30b440342f0698884bc98c2382c80ab", - "sha256:047b65c59356e59a9f1d90350f057ec2ad17c5bbb25bd18cefdec7318e7a881c", - "sha256:04cceb79d9b962c80bfd583ca76364e984c769d8ef47aa56443c4c1d88a16376", - "sha256:08de9fe0bb8bef7b06502ed914431bf708fe35aa1690a14df64825ad3a67fd0d", - "sha256:0a7ced0ad7f88105519ce2dcca97b95ee97516e8461657ff825c093dc680b8fa", - "sha256:0ab47e8c409afd83d4d21ecab0e81bd05930bac2035936d7b670720fa534fa4d", - "sha256:0ea3c2585385fe3cda86e5720cd8ab8fb6fa40e0161339cc3c0bfc35e2312a7d", - "sha256:114ec9dffdae87dfbf88f9e0f1b7b9333973bce4af0f0cc71537b1ebf098fe02", - "sha256:1262330d8a3a358625397269925620d7ded2e1ec531ed49181a84954d5cc9054", - "sha256:12d210257911b3ab4c0f83c8de537d2ca2489bac12d7ce860b5ae6e9197f742c", - "sha256:131f8ec69ba6fe1c8a78236d9ffbf3cf68d4125134dfcabc3305df2d791384e8", - "sha256:1bd77a327200a244f5179fcb3f7674a1588cbf95da0368265d50fc70942c3ec6", - "sha256:1d0388fc101e51f19512d1c204202a96d7e069489369ba2ffc31596065156ccf", - "sha256:1e4bcd812105890bfda1809d096472fec223da9145aff7f508eb2bc770ca3e00", - "sha256:1f5563a21096bee6fadd714e175f6b4f1f06747f8e3aea09f2f747889df11f54", - "sha256:20530b025d5d5b118aac1a239011ce8387242547966438c4744f8a25efe1bfa6", - "sha256:22fb529ee55983ef3c46b11771d3d84f58ce3216deaceb303b18b290eae66608", - "sha256:28a237c01b9d5fa2120ee4606e1ca3a989f55b67698458b76a793c1a48671887", - "sha256:2a71354c2da18cd117a88d1bc983bf6287abc465ccd5357360091c1d4726161b", - "sha256:2db844b990c2160d42dc6ed1765e9de68a319bd67d1bcbbf8f71a7a6be013c40", - "sha256:329621ab7581d6fc806b60d1fe88e48a4f568d44a2406954cf6fdb3344e83d2e", - "sha256:3313708237df9efd58092915fef3ee222585c02a12051f06616566db0999559f", - "sha256:350986411597c90cc257010fcb451261716d0814ca1cb6882f2cc303a17cf61e", - "sha256:3938b80bb38a5370d34659e4e16f2be7750854ab4f2b524e7a534b610b93ae91", - "sha256:393a9cf2f2c0f79e1575e9a1ff0cb460337b2bdc7f56b068237d4f1a269e03e6", - "sha256:3a42be69b137494f26219674d38ca3801e5db2fe60de02c332f19d6597729544", - "sha256:3ac5f8f9a1ed671b126975b28bd0696d033c701fd24ebaf18c2a4af38b7db183", - "sha256:3ba113b254cb22406199a72592fa14acb9713128cedaa6156153fe8cf66d7417", - "sha256:3d4f8f746521aa6f9d1d1bfacbf6087449cb8946139f5ce50e7a070db96554b3", - "sha256:3f6fe03aaceca011d151b462084b37f3001f6279cce04537319876e432d586f1", - "sha256:41637188b8d14fb75140ca4d569c7c2de10f025ba89960753766b1c253dc6f79", - "sha256:448cc450e540da6650100a462dc5acb3e6972de82d102ee94015d3ab47071dbc", - "sha256:46d809ff965a285b3890ed65aaf84f75876cb5d568ccbe6f1f58a1a3a44c4b94", - "sha256:46df6d133d089b22a5ff53e542851631a09e69a022f3822c62f9dfe0e558c8b9", - "sha256:487aeea9781651a941f32107720691e2cdc6ec51ef4a97a9e2ea793b0ef66a80", - "sha256:4c238565d5a1321167cbb7b5e818bd07fef56b46896c0a05a9a3c5db0d3b5bcd", - "sha256:4ccf312f7ed82581713ffa502420dc8715ee908325a8ba3e53490c5da578720c", - "sha256:4e33686827f82549c100cdb70b9a301e1dd77593770dd7a2f9c9775b39d779cc", - "sha256:4fae096efc700ae2d5269dae1d008b6c841f9044f1090c475ab4994ba19d5cfc", - "sha256:5086b1504e715647fa608d6dc6b801a273d14e68a52d1c48a55feab125f1a065", - "sha256:5494d247cd6df5fc4c4115cdb4254ba1530c0d4b67e9a228b401938eed8565ac", - "sha256:5707da68092852ed75e01c302eaa161b8dbf04c06cb78e574c5142ecdf26f358", - "sha256:59985cfa7f655c97ad29f6c6f10e2d052a90809eda79fe9504107cdaccee3448", - "sha256:5ff9c64d53c551cba31617510600907b85c706457138e2b4770bb8cea722c0d3", - "sha256:6050dbc9dcaeb389099c7ab153296237e9f0ee89705b96a7d5b936c96a0fab04", - "sha256:6097021b35ff52e86a4a476acea7c0efd71a9483b9cfc6b15b0d4c1f8e2f1fd7", - "sha256:63950d0ba16d1f89a8edbdebf2b82bcc5e724e813f0c0f4c7f109b823fecbf87", - "sha256:6771d2930dea96be6f6dc60bdd21db77b4dea2f8bd09752c013b1f6241c7e112", - "sha256:6812683714b3f8c7048e7c441b20cc4ef6213a2335486d555e4217f0726c7b55", - "sha256:698e05518bf193de0b76388f2136889840f28effb34768ca4a763523e6ee9245", - "sha256:69ab30ac5fb24f943416884b7cd88515e5cdfe875d25cfd4edea04ba6a6f8c81", - "sha256:752b752c0d40497a6702533a57c2739defe94765b563eddc7a929d8d3e1a471b", - "sha256:76899f5b699e32c49475231e90ea8cea61321a4462dc44bd10c1805ce45b0376", - "sha256:778d7d81d4233f26bf53ea27fed6f2af10216d2e81706c5e952461c454a264dc", - "sha256:7a9900ef63b202746210f2ba86706accc4d86c6a47ad0cd0a229b8800230ec94", - "sha256:7a9fe7e7471c40043ca9d8a85d2966f6375b0d42735a42266f9eb4e3afdcc33e", - "sha256:8667e1d9faae729d39c409c649f6b83244eb88690e362f78461ec20d52afc7a1", - "sha256:89c96e1ba5ae1ddec0655589e343c09826d4ea4118deb4b4173f6f971999dee8", - "sha256:8c162c5e145a2975e45e77b60e2b3340252df49507153def80064a10696b513a", - "sha256:8c49d65d207338b8dfe6cd0b788c7f86d5c61dd7c027c155ba3db4ca0a4ad948", - "sha256:8d9612112fa5311a38625c6f61d28ea2ae48a12d739ffef12bbfc42c0c2a2dde", - "sha256:8f0c58dd702928aedc0072fcb43dd235f84cebccc689064cdf2f935b6a154d4b", - "sha256:8fb2eec707276804abe731e918fffc127fb43ba7497741bf274ef5bdb5c077e8", - "sha256:900135e33d9260ab929cd8f5a88e4a2458eef7b34a87aa386d6c39d05923bb9d", - "sha256:911d5f61fcd56ac7c6bfc4568fdf2c4c1c3a9e2a8976a3274166a03e35249e28", - "sha256:917ad33e2b4968e28bf780033a188736a99946a9d91df1be8eae60256c82ace6", - "sha256:91e92044bcb6e7d4860336cac197f487ecbc433bea443e2e0e90a8dc692ca16c", - "sha256:964cc82002c91b3acc25b9b77da7981885e273a6cd601c1f15e732aa1ac744be", - "sha256:97db0360c94674657217b0e0bd83cae85a820ae91030729869c3f955a7b056f1", - "sha256:98165faf913868f66afaa2f3293ad69bef36308e8a75480a0567bc35738c4cc3", - "sha256:9b5aec9e01f4c9b528d96320fdf560398ecd6f6aa65476390e5b8dc7e2cb701c", - "sha256:9e576b4f9670412c7cac8eefa1a969aecb3d1879bf6daca2560244cf795563ca", - "sha256:a2c2fa250e7acef8712d842c1e5ac765da7f12084813867f168ea83b4bd5cda2", - "sha256:a341e0200a34603ee81deb124de597e6c723d67171ca058966945bd68d77a6d1", - "sha256:a5836dfc928e197f53126d1ee89f5b73f194218a28c3aaf7ec0d8e1ba886f147", - "sha256:a608474e07b28adf954850a2be7d1ee25572d6dc90d1323bc0932dc9b807ffdd", - "sha256:a8e700f49d6174c06637cab489cc593c13223181e2c00bbec8de782140743144", - "sha256:ab7cd8a404a086b30c671c78377a0806dc113e676a4af80c85323f0bb2f85300", - "sha256:ade4d7954a18560529aa1bc0930b0dfff5a3fe5e6772a3a0810307c09d61af31", - "sha256:b30fc99e69971b91ae80c37b87c4e66dcb82d968bfe3a59b37bcae0f7a58bd49", - "sha256:b9c1bab76af86908aa1115b7b8fa5e65ebf0d0adc7d9fc8e27a3bc8a14dc6202", - "sha256:bc60251893baa93f3c4b292764b17b8ca6fbb50972de9f7fcc248634aea2fade", - "sha256:bdbb7da34e9445915fd4ddcd53a360655d3f472b4bfa52b662b2f938573d53ab", - "sha256:c704216ffd31b5eec26cd998117fd24d33c65f8b5930faf200257670bd3ff1f3", - "sha256:c709f4b02492c28936bca9e2f99a554f7fb527bdd76f2b17ffb4fd0b0107d807", - "sha256:c817ec812a21a7d3fa7b772aa56704a4cf4d8f79453600c4e30b2a4a163a1f85", - "sha256:c9424f15818595de1cf6f7f9ec4e01848d6ff8335aaaed3aee4836c2a4776312", - "sha256:cb769206eeb69df28de4eb4a38480ff07dc7ef236a8ed6784908d45db7b1946b", - "sha256:cc5cfce70b40e4ccbcadbeac3fc10c14db18766822c9d671f6d9cbc7c6dd031c", - "sha256:d21b6f1710125cccc5d5c7be4fa9d06007538ab4af1fab5c0028e286fb99447a", - "sha256:d30acdacb94cd6da75569ce9b5c22bb04dec35e4411e4b483f2d683d0d4c8cd3", - "sha256:d400e56a203fcfd9275b11e08114b5dc17925ad58fff84a9ac4ec37a6a611ad4", - "sha256:d7e24f5729d4d09e50a9594633a600e51f1423aa685706ca87b054cb2f220df3", - "sha256:d9b7b622bc7e47854ea660683d4722e6057a5a07176f38c54ce5da8ea7b53741", - "sha256:dbe04ea4a9aa1d4210bf1e8968f94695acad72ef02626c1cfdef9a4539560788", - "sha256:dc5abb94be9f670e06622531042c5d3f1c1fd338c872ed2b2c9a164bdbfdf75a", - "sha256:def88ea5e95c202f89a60809c0ba13919f9e91ac6e94d8fa0cfcc6e13169fbd8", - "sha256:e3c18a6e8ce823ac113f3e69c220bb5137ad60eff4ffe01c56d42dc8fc0bffa0", - "sha256:ed3f4f39d219b7f304aec0b28a95c273111b69ffcb5b87d61b1fb3601dd964f9", - "sha256:edbb829d5b7864385fbdeca8c0b221249a9fcc4b852d0d4f92db60f93251dc9a", - "sha256:ee9f9736fcaec21d182fb2fc359c8ce4c92e606b0f2b60c8c02aac8e2b345666" + "sha256:019ae21de930d6077efa1eac746de4df5234e7c6c11ab10080c0935fc5abbecf", + "sha256:02688fff6d256afdd57da5359144ddab8e054b2ba98ddcf147fe191bdf996e88", + "sha256:0274b87df89d1dda8dce77cf05a9dfab7bd30045a09e0d9435ec8be622e374e6", + "sha256:0323e8dbeec4d63c27111796baa7e8a89b391c32d90e67d78f9404d0c8edeab4", + "sha256:053edbb52fe8b8a1a6698c4fee39590c9e44a602ace807291eb87e3b17f85f48", + "sha256:059027f5dd2aafb916301f46a619c7fe03ff5761cdb2d091cf80bf6dbc24bc29", + "sha256:05f11a4be4f668974238cff21208fbd9f629cab8a68b444b7d4a4cfd8081b1d6", + "sha256:0ab71cc5ea86f6685a7b2235edad65f1f2a4b6341109af259d758973d96eece5", + "sha256:0b439f4fb0b615bc0443cc83eaf5835bd480f680c69ed1be963bdb401b8159f8", + "sha256:0ec50d24a12e50857e94ac9035d3c06fd0827bb477b9ebcd83a2a49dd89e5e23", + "sha256:131fc50d52a52acc367ea8bccb028447b734243d00ba1cfc7d9ff8d0dc37fa38", + "sha256:17b5f1d1a4a5ac536283298c98cafc5632ae3897c8601fb2ec8babc6f47a1be9", + "sha256:183b8da9b870ad171a11a629c43e0587a228aea9d595a969231d59bf530b6c77", + "sha256:18888d50813b9df9b8dc8c1506ec40c783db25f130a6101eb89896b27076f751", + "sha256:25b88277832eb558305c3bb986ad61f19b5cb5a87aced289bce4a1701a92aa31", + "sha256:266cdab48e2242b6c010beb8b7af4164aa87f4ad8d6fbd9f4f531214f8ddb234", + "sha256:281bffb09b2e1620db4e99a9df96e38d939c341c7c43cd5191326fbdb4d42275", + "sha256:28cd002cf5a499e6e9bd69d992ffd501b8473948f3e97d6e075b774df1901e8e", + "sha256:2972c6c6a806e0c788f6ec39510abdb61b3a648fd141a5fa77becd2cc05ff551", + "sha256:2b4027b370cc46c4802ba32a979729209c0407d548723e809f19a50a9df27405", + "sha256:318c924e218be754427ce6bb4c630d9dcb5478eb00a8a3f8a0972086adc763b1", + "sha256:380accae56f8c9df99f34bc7e79d286fee37c3dd06b362c394b08ea96371b7c5", + "sha256:3c7784f9936292c9d3f92fc772d874edc071a16cd883ea0d997e5c4318f6362c", + "sha256:3ebd85fd6253abe89f852fc008294d490eb7a5f66913703148b8d263b048cc90", + "sha256:4126c8fe9d817ac3ab223ee5db41a09d0fa82dbd6bb59d207b6f7313d733f19b", + "sha256:4155f0ab246b6892110960f25989ab91073cd708b974f4732dca4d219a8be3e1", + "sha256:41f16267d8e6d916e06a6a1a0e151f643a6bab1277945a4bd494f359d4185dd2", + "sha256:4522f5d662d3ee55a072fad18e2af5dae480658d4e23b04b455c4b7542ce4327", + "sha256:46c900c807b0614c454ba89271ec6f59212403c54dc68ea493ab1ece2c510618", + "sha256:48291b25a904243f37c9aabbfed3eaba466c9a993f5f5946fe647163b7face07", + "sha256:5038a5e9e106087c117f0a7d6fd9d8a382b228da24bbd085b9f2b5d54ab11c3a", + "sha256:594a26bcf0cb720c16ac6db3fd4b3f411be756f9da7682f2f629089ff15aef18", + "sha256:59706135d3107939effe9f9263bd78c507f4abd7bfb96acc5a7f4176aa0a90d2", + "sha256:5a327d7581696c7a392a8f85cce7e54fa1303f5b79b3b2983abaab309b56cfd6", + "sha256:5eca8a45d38c916783c44e5da06a367b77234efa51d84dda8804654b99efecc9", + "sha256:5fa85f6789178ede5333568cbee5bac5fa9718d5f02406b65545e83368fa8fe9", + "sha256:65097e45ef7a942a9b92999b81d2e91fe80cbd0616215e625af39d2166692018", + "sha256:65cc9938cb9bd8862fc220e0719fd7f9c291d788f0a62bb8840820c46fa5a4d0", + "sha256:6a4c3607e2a0e66337d8ddf95ca7efe9b30ebf944119a4fb86503ea66f777263", + "sha256:72f11a136f148eb1218e7d1492749b8b5594302010db0cebd47423c4ac8c79ee", + "sha256:78b5a71de59e30c697a64c69fc48b032bb99c43b7437091b808a9ba20bb0235c", + "sha256:7b212edc9bf9d0c25cc3117483289b9e1a49a1ed134a02635baa987e9f0d89db", + "sha256:7e0f7045c420abdea249a28384baa846b87bad5c9f42af1957dc50c6e337fa1a", + "sha256:7e83cfec424f546dc3f0cc71896f8cc384a711f4116bc1abb0598302a9af3240", + "sha256:80c55bcc31d21bd07f7d1589e11f2ac1faf3359cf9f93026a1944ee76a40f954", + "sha256:863740d7f45adfd29b95658a680b16113721eaa89857c67e7e9573c61e87bbd8", + "sha256:88484b8c3f71dc9205d0d36da541e2cdcf4bc74474a2ee8d99c2e6411b659b89", + "sha256:8a08810e0bcc606d10cf1c5389c96fc92362244c0cf761358c495c2eb29df3dc", + "sha256:8c0637ae4fcb54d5c7fc9af24d348003b6f9dbaf7a06bf13f769d7b85903af39", + "sha256:8e9e3409338a42e3d4c30c224fdb678364542c77994f089fd6cc8131969eff48", + "sha256:902ea10ba85e014dc5d23a7bbb3ab70722349561e73783dd71571359e8867244", + "sha256:9533db74a2685169380db3db3ab59643453e7c486fffa9bf3ab60b73c4e174be", + "sha256:97f02ff49d1fa21308207a7743bec4fdd7aa90e8dd091539da660fc51e624c4d", + "sha256:9ea9a2a154dc7d8658930fa87cda0e6094235b5e130f037d9894eaf8722119a5", + "sha256:a0440d847b2c9986e4d27e8a59164714e5198530c69a5f9fb2e4620f9136d653", + "sha256:a6d39a27b542a781d691827b955d685d496fb6cccfc6eecc336a78b399032062", + "sha256:a7f4d3c478b1fcf412bf6c82914b02fed33ab359120df9172dda7bc855227461", + "sha256:ad297807bbdffce61b04e5e0c22f3c5d9e1905c1ee186f1f6d029f83bf0f18b8", + "sha256:add6778bb51efb80174937543754d2dfa0f4e504e7302d97896006a642c14f95", + "sha256:ae075ebf7bb5f48b3bd2fc9cd53346e4ff43e2515a4f822914bbc62a3cbd6e7e", + "sha256:b26fb439a7fbb522af63bbd781fbf51ec0c0659134a93f5bc8e9e68641df811e", + "sha256:b2bac59721d246939b21274229b9923aeae3db97b6118da739c658c17e110dd6", + "sha256:b314ad1f0667715e8d1b6197d5336ab579b13e801172721d62331bd40034a30c", + "sha256:b7317035875bd7c4705e2566848b2043b78e18f2f5675ea651f9f7805b5589eb", + "sha256:b8e936e620e5f336a207e08c0da9dace5d4dbcc8e64743ab1acaa77a64bbf060", + "sha256:b906da4e9a7ba4ec33ed2f7238343866932c1a6f84944c804252b2922708d0ee", + "sha256:ba690e4e33c360fcf0b8411ca90f8b9cc595e8deddd6a25a9a75a725b698cd6a", + "sha256:bb14da3d63da994c34cfa47cde469df8013ddf5f575455a22530c8c4a0ed8616", + "sha256:bbc2e1632f4a61fa171ddab3bc8368fb8475e7ce68733ca92fec862fdd8e0f60", + "sha256:bbdd3c896db09993b7879cd35e56da6ed8918d161d6e80f9d9c40d78d34e4784", + "sha256:bcaaa8e542cb7e1962d0a58ce6a25f6b4b6ca2e5ce743155fc1f6eb2fea52574", + "sha256:bee682ab1005aff597946234e47c95fcf0f44d2b1f38075f0aba26bbc4e7545a", + "sha256:bfec6543d60c57e7543d9cbccdd5dfcf562f2c05cd6b814df68108a20794e254", + "sha256:c2e50baf7be8831524a87beec6c1873539519a1948f907dc3d4b9be27ebacb80", + "sha256:c6c79a6138be017d85f3bab1df735669b669a38f9b3ff646a1f179afbacb7b63", + "sha256:c702fb7c8bfd87c9ce9c8bddfc9a5796a492bab35a52b1693adee413721e32f2", + "sha256:c9ba1725826f6571a6e4c1561bb1613711f0058b91927a147dc42c637ba087d9", + "sha256:cf205ac52cb6b45745c0a4891cdb6e709c10ad5b034aa736aff561fc4ce9828c", + "sha256:d0d03fc67499ee90feedfa2add4aaa1c091a7bf333535d847b10fffe390e58fe", + "sha256:d118d63f08fd6ac285cb8166e96c992a6ed0e7a1644e8790c39070b18779e688", + "sha256:d24c09f397c3ce55f20e0250da7ba5b0e5249cb5d21465e71ec15154a3a7e8e0", + "sha256:d41735c7a646dae8612e0552dfc53f45807eeb54364dfb1f0a65ac274bc56b3a", + "sha256:dd1696d91f2a37cece9bd22e507e7be7c37c59ecc61fd15f0d0f31e3b6888957", + "sha256:dfcad9c63a893c95ba1149481b9680ce68dd71211f08df0073ee62700790bc97", + "sha256:e384782608837d9aaf123e413679883091744664a2cd76f0ad0e0a1f12facc57", + "sha256:e5ea0abea338c617b753082f36f64c70ade853d88e91ab5732b301ae8ed16e3f", + "sha256:e6ff81c570413bcc35f1c16850eb66e2493a3259e68efe8672376533d2c82d38", + "sha256:e88951ad2831880405f3f055ab12a6aa72696c20a2815128eeccdc3bf914cd78", + "sha256:e98e16b6ce531b12100c01daac922e8ec5b991832a5f58003f13b7d45ea82dc0", + "sha256:eb0fd32e8e433797499571447d9f975b4744be79c0a3339413868d79517231ed", + "sha256:ee74a73e1f9e16b71f67329e99bb58aa4af9a2c3c4b3a5db9f26e92e7c39e161", + "sha256:f15ec5f825c283a5aa427d78759ab8f84e7b5441d15cfff476b548bce3764666", + "sha256:f296c7fe928ce0e29e313f85c43a5ab80542e096e1163c2605b8cc18aa2aff2b", + "sha256:f32df1b19f773bb41382e8b215955d248c9766e3d6ff5a1dd89709e7d96e4685", + "sha256:f3ed67279a4b317a808ac743d3a915f74187530c5f3d9c859e5d04d475b8c174", + "sha256:f5b972ca514898fb7131671c425a62ca38fdae2a8d6296e4b605ec8202349f8c", + "sha256:f961086c0dbba6c00cbd5c5b5646247efd0d0a4044444bfaa9efc7a6ba5e96a5", + "sha256:f9bd7d7a449667d6f17edd9045ec82a4ed2767afb91743d3d0b18c376a56dfe2", + "sha256:fbac4c8ffadb685189efa92fafdb2f5392e9cbd262eae3818bcdb1bd19acaaf2", + "sha256:fc43c8276d0a7c7b76f31d4f3f80f9eb820673628f1411770a70029c1d5f6a75", + "sha256:fcfded324f0710632e22050a2fd7b56b1cbcb2d21001630bcc26d536f54bffec", + "sha256:ff435abdcbfdf4a070f488830cd53aef77cf8649d0fd8ed76bf27d9566e80e78" ], "markers": "python_version >= '3.6'", - "version": "==0.20.5" + "version": "==0.20.7" }, "lxml": { "hashes": [ @@ -1195,10 +1195,10 @@ }, "python-levenshtein": { "hashes": [ - "sha256:114dfa996d5aa91d400dd816210a62523be8c598962aebbf7b086c7fd26fb72d", - "sha256:f55310dc28bb891428d751299688e0671fa3e0405ddd7561884747d6b1be2e7d" + "sha256:88a58b95e3340a918489dac0c78f731323c0a4d8f5564f839ffea80155574e77", + "sha256:9228af5523f797f0798f045dc4a95ed1f46df72bc2186e52b530a33998a51b37" ], - "version": "==0.20.5" + "version": "==0.20.7" }, "python-magic": { "hashes": [ @@ -1654,11 +1654,11 @@ }, "setuptools": { "hashes": [ - "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012", - "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e" + "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17", + "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356" ], "markers": "python_version >= '3.7'", - "version": "==65.4.1" + "version": "==65.5.0" }, "six": { "hashes": [ @@ -1736,11 +1736,11 @@ }, "tzdata": { "hashes": [ - "sha256:74da81ecf2b3887c94e53fc1d466d4362aaf8b26fc87cda18f22004544694583", - "sha256:ada9133fbd561e6ec3d1674d3fba50251636e918aa97bd59d63735bef5a513bb" + "sha256:323161b22b7802fdc78f20ca5f6073639c64f1a7227c40cd3e19fd1d0ce6650a", + "sha256:e15b2b3005e2546108af42a0eb4ccab4d9e225e2dfbf4f77aad50c70a4b1f3ab" ], "markers": "python_version >= '3.6'", - "version": "==2022.4" + "version": "==2022.5" }, "tzlocal": { "hashes": [ @@ -2137,7 +2137,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.1" }, "click": { @@ -2259,11 +2259,11 @@ }, "faker": { "hashes": [ - "sha256:245fc7d23470dc57164bd9a59b7b1126e16289ffcf813d88a6c8e9b8a37ea3fb", - "sha256:84c83f0ac1a2c8ecabd784c501aa0ef1d082d4aee52c3d797d586081c166434c" + "sha256:096c15e136adb365db24d8c3964fe26bfc68fe060c9385071a339f8c14e09c8a", + "sha256:a741b77f484215c3aab2604100669657189548f440fcb2ed0f8b7ee21c385629" ], "markers": "python_version >= '3.7'", - "version": "==15.0.0" + "version": "==15.1.1" }, "filelock": { "hashes": [ @@ -2608,11 +2608,11 @@ }, "setuptools": { "hashes": [ - "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012", - "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e" + "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17", + "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356" ], "markers": "python_version >= '3.7'", - "version": "==65.4.1" + "version": "==65.5.0" }, "six": { "hashes": [ @@ -2631,11 +2631,11 @@ }, "sphinx": { "hashes": [ - "sha256:5b10cb1022dac8c035f75767799c39217a05fc0fe2d6fe5597560d38e44f0363", - "sha256:7abf6fabd7b58d0727b7317d5e2650ef68765bbe0ccb63c8795fa8683477eaa2" + "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d", + "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5" ], "index": "pypi", - "version": "==5.2.3" + "version": "==5.3.0" }, "sphinx-autobuild": { "hashes": [ @@ -2739,7 +2739,7 @@ "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" ], - "markers": "python_version > '2.7'", + "markers": "python_version >= '3.7'", "version": "==6.2" }, "tox": { diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index abb3d3dfe..405df07ce 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -249,16 +249,22 @@ class RasterisedDocumentParser(DocumentParser): if mime_type == "application/pdf": text_original = self.extract_text(None, document_path) - original_has_text = text_original and len(text_original) > 50 + original_has_text = text_original is not None and len(text_original) > 50 else: text_original = None original_has_text = False + # If the original has text, and the user doesn't want an archive, + # we're done here if settings.OCR_MODE == "skip_noarchive" and original_has_text: self.log("debug", "Document has text, skipping OCRmyPDF entirely.") self.text = text_original return + # Either no text was in the original or there should be an archive + # file created, so OCR the file and create an archive with any + # test located via OCR + import ocrmypdf from ocrmypdf import InputFileError, EncryptedPdfError @@ -276,9 +282,7 @@ class RasterisedDocumentParser(DocumentParser): self.log("debug", f"Calling OCRmyPDF with args: {args}") ocrmypdf.ocr(**args) - # Only create archive file if archiving isn't being skipped - if settings.OCR_MODE != "skip_noarchive": - self.archive_path = archive_path + self.archive_path = archive_path self.text = self.extract_text(sidecar_file, archive_path) diff --git a/src/paperless_tesseract/tests/test_parser.py b/src/paperless_tesseract/tests/test_parser.py index 700782a92..858cc7701 100644 --- a/src/paperless_tesseract/tests/test_parser.py +++ b/src/paperless_tesseract/tests/test_parser.py @@ -341,6 +341,17 @@ class TestParser(DirectoriesMixin, TestCase): @override_settings(OCR_PAGES=2, OCR_MODE="redo") def test_multi_page_analog_pages_redo(self): + """ + GIVEN: + - File with text contained in images but no text layer + - OCR of only pages 1 and 2 requested + - OCR mode set to redo + WHEN: + - Document is parsed + THEN: + - Text of page 1 and 2 extracted + - An archive file is created + """ parser = RasterisedDocumentParser(None) parser.parse( os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), @@ -352,6 +363,17 @@ class TestParser(DirectoriesMixin, TestCase): @override_settings(OCR_PAGES=1, OCR_MODE="force") def test_multi_page_analog_pages_force(self): + """ + GIVEN: + - File with text contained in images but no text layer + - OCR of only page 1 requested + - OCR mode set to force + WHEN: + - Document is parsed + THEN: + - Only text of page 1 is extracted + - An archive file is created + """ parser = RasterisedDocumentParser(None) parser.parse( os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), @@ -395,7 +417,7 @@ class TestParser(DirectoriesMixin, TestCase): - Document is parsed THEN: - Text from images is extracted - - No archive file is created + - An archive file is created with the OCRd text """ parser = RasterisedDocumentParser(None) parser.parse( @@ -408,15 +430,26 @@ class TestParser(DirectoriesMixin, TestCase): ["page 1", "page 2", "page 3"], ) - self.assertIsNone(parser.archive_path) + self.assertIsNotNone(parser.archive_path) @override_settings(OCR_MODE="skip") def test_multi_page_mixed(self): + """ + GIVEN: + - File with some text contained in images and some in text layer + - OCR mode set to skip + WHEN: + - Document is parsed + THEN: + - Text from images is extracted + - An archive file is created with the OCRd text and the original text + """ parser = RasterisedDocumentParser(None) parser.parse( os.path.join(self.SAMPLE_FILES, "multi-page-mixed.pdf"), "application/pdf", ) + self.assertIsNotNone(parser.archive_path) self.assertTrue(os.path.isfile(parser.archive_path)) self.assertContainsStrings( parser.get_text().lower(), @@ -438,7 +471,7 @@ class TestParser(DirectoriesMixin, TestCase): - Document is parsed THEN: - Text from images is extracted - - No archive file is created + - No archive file is created as original file contains text """ parser = RasterisedDocumentParser(None) parser.parse(