From a823b8f70ca7c19a9916d6256351d7fc2892a2e7 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Tue, 3 May 2022 11:15:31 -0700 Subject: [PATCH 1/5] Includes a version.json file with the current version in export. On import, catch certain errors and check the version if possible --- .../management/commands/document_exporter.py | 9 ++++++- .../management/commands/document_importer.py | 24 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index b110475a5..27f1fd8a9 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -22,6 +22,7 @@ from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME from filelock import FileLock +from paperless import version from paperless.db import GnuPG from paperless_mail.models import MailAccount from paperless_mail.models import MailRule @@ -232,12 +233,18 @@ class Command(BaseCommand): archive_target, ) - # 4. write manifest to target forlder + # 4.1 write manifest to target folder manifest_path = os.path.abspath(os.path.join(self.target, "manifest.json")) with open(manifest_path, "w") as f: json.dump(manifest, f, indent=2) + # 4.2 write version information to target folder + version_path = os.path.abspath(os.path.join(self.target, "version.json")) + + with open(version_path, "w") as f: + json.dump({"version": version.__full_version_str__}, f, indent=2) + if self.delete: # 5. Remove files which we did not explicitly export in this run diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index d1ae33afb..398bc05b4 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -6,9 +6,11 @@ from contextlib import contextmanager import tqdm from django.conf import settings +from django.core.exceptions import FieldDoesNotExist from django.core.management import call_command from django.core.management.base import BaseCommand from django.core.management.base import CommandError +from django.core.serializers.base import DeserializationError from django.db.models.signals import m2m_changed from django.db.models.signals import post_save from documents.models import Document @@ -16,6 +18,7 @@ from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME from filelock import FileLock +from paperless import version from ...file_handling import create_source_path_directory from ...signals.handlers import update_filename_and_move_files @@ -53,6 +56,7 @@ class Command(BaseCommand): BaseCommand.__init__(self, *args, **kwargs) self.source = None self.manifest = None + self.version = None def handle(self, *args, **options): @@ -72,6 +76,11 @@ class Command(BaseCommand): with open(manifest_path) as f: self.manifest = json.load(f) + version_path = os.path.join(self.source, "version.json") + if os.path.exists(version_path): + with open(version_path) as f: + self.version = json.load(f)["version"] + self._check_manifest() with disable_signal( post_save, @@ -84,7 +93,20 @@ class Command(BaseCommand): sender=Document.tags.through, ): # Fill up the database with whatever is in the manifest - call_command("loaddata", manifest_path) + try: + call_command("loaddata", manifest_path) + except (FieldDoesNotExist, DeserializationError) as e: + if ( + self.version is not None + and self.version != version.__full_version_str__ + ): + raise CommandError( + "Error loading database, version mismatch. " + f"Currently {version.__full_version_str__}," + f" importing {self.version}", + ) from e + else: + raise CommandError("Error loading database") from e self._import_files_from_manifest(options["no_progress_bar"]) From dd4d903f695954bb8c3168fbbfa8a6abae0c0cfc Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Wed, 4 May 2022 19:41:49 -0700 Subject: [PATCH 2/5] Uses the correct styling for output messages --- .pre-commit-config.yaml | 2 +- .../management/commands/document_archiver.py | 2 +- .../management/commands/document_importer.py | 22 ++++++++++++++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0bf9bace..fb824db9f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: exclude: "(^Pipfile\\.lock$)" # Python hooks - repo: https://github.com/asottile/reorder_python_imports - rev: v3.0.1 + rev: v3.1.0 hooks: - id: reorder-python-imports exclude: "(migrations)" diff --git a/src/documents/management/commands/document_archiver.py b/src/documents/management/commands/document_archiver.py index f33ccd7ce..bb376c2dd 100644 --- a/src/documents/management/commands/document_archiver.py +++ b/src/documents/management/commands/document_archiver.py @@ -152,4 +152,4 @@ class Command(BaseCommand): ), ) except KeyboardInterrupt: - print("Aborting...") + self.stdout.write(self.style.NOTICE(("Aborting..."))) diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 398bc05b4..18bbe6c7f 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -80,6 +80,18 @@ class Command(BaseCommand): if os.path.exists(version_path): with open(version_path) as f: self.version = json.load(f)["version"] + # Provide an initial warning if needed to the user + if self.version != version.__full_version_str__: + self.stdout.write( + self.style.WARNING( + "Version mismatch:" + f" {self.version} vs {version.__full_version_str__}." + " Continuing, but import may fail", + ), + ) + + else: + self.stdout.write(self.style.WARNING("No version.json file located")) self._check_manifest() with disable_signal( @@ -110,8 +122,12 @@ class Command(BaseCommand): self._import_files_from_manifest(options["no_progress_bar"]) - print("Updating search index...") - call_command("document_index", "reindex") + self.stdout.write("Updating search index...") + call_command( + "document_index", + "reindex", + no_progress_bar=options["no_progress_bar"], + ) @staticmethod def _check_manifest_exists(path): @@ -154,7 +170,7 @@ class Command(BaseCommand): os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True) os.makedirs(settings.ARCHIVE_DIR, exist_ok=True) - print("Copy files into paperless...") + self.stdout.write("Copy files into paperless...") manifest_documents = list( filter(lambda r: r["model"] == "documents.document", self.manifest), From c907d690b7dff2d5ca7c019026ad1013df61329e Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Thu, 5 May 2022 09:17:51 -0700 Subject: [PATCH 3/5] Improves the output to the user --- .../management/commands/document_importer.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 18bbe6c7f..92742a487 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -70,13 +70,13 @@ class Command(BaseCommand): if not os.access(self.source, os.R_OK): raise CommandError("That path doesn't appear to be readable") - manifest_path = os.path.join(self.source, "manifest.json") + manifest_path = os.path.normpath(os.path.join(self.source, "manifest.json")) self._check_manifest_exists(manifest_path) with open(manifest_path) as f: self.manifest = json.load(f) - version_path = os.path.join(self.source, "version.json") + version_path = os.path.normpath(os.path.join(self.source, "version.json")) if os.path.exists(version_path): with open(version_path) as f: self.version = json.load(f)["version"] @@ -91,7 +91,7 @@ class Command(BaseCommand): ) else: - self.stdout.write(self.style.WARNING("No version.json file located")) + self.stdout.write(self.style.NOTICE("No version.json file located")) self._check_manifest() with disable_signal( @@ -108,17 +108,24 @@ class Command(BaseCommand): try: call_command("loaddata", manifest_path) except (FieldDoesNotExist, DeserializationError) as e: + self.stdout.write(self.style.ERROR("Database import failed")) if ( self.version is not None and self.version != version.__full_version_str__ ): - raise CommandError( - "Error loading database, version mismatch. " - f"Currently {version.__full_version_str__}," - f" importing {self.version}", - ) from e + self.stdout.write( + self.style.ERROR( + "Version mismatch: " + f"Currently {version.__full_version_str__}," + f" importing {self.version}", + ), + ) + raise e else: - raise CommandError("Error loading database") from e + self.stdout.write( + self.style.ERROR("No version information present"), + ) + raise e self._import_files_from_manifest(options["no_progress_bar"]) From 915382f2c77af1ddf5e95d806aca52d5e5588def Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Thu, 5 May 2022 09:23:15 -0700 Subject: [PATCH 4/5] Adds documentation note about version matching --- docs/administration.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 1139c3a69..f6b0ed659 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -287,6 +287,10 @@ When you use the provided docker compose script, put the export inside the ``export`` folder in your paperless source directory. Specify ``../export`` as the ``source``. +.. note:: + + Importing from a previous version of Paperless may work, but for best results + it is suggested to match the versions. .. _utilities-retagger: From 3baf29a6b74e9bb6942843653cf2f50d8048efc7 Mon Sep 17 00:00:00 2001 From: Quinn Casey Date: Tue, 10 May 2022 17:41:09 -0700 Subject: [PATCH 5/5] Use currently... importing... instead of "vs" --- src/documents/management/commands/document_importer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 92742a487..1a1316059 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -84,9 +84,10 @@ class Command(BaseCommand): if self.version != version.__full_version_str__: self.stdout.write( self.style.WARNING( - "Version mismatch:" - f" {self.version} vs {version.__full_version_str__}." - " Continuing, but import may fail", + "Version mismatch: " + f"Currently {version.__full_version_str__}," + f" importing {self.version}." + " Continuing, but import may fail.", ), )