s/Sender/Correspondent & reworked the (im|ex)porter

src/documents/management/commands/document_exporter.py

@@ -1,10 +1,12 @@
+import json
 import os
 import time
 
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
+from django.core import serializers
 
-from documents.models import Document
+from documents.models import Document, Correspondent, Tag
 from paperless.db import GnuPG
 
 from ...mixins import Renderable
@@ -14,21 +16,19 @@ class Command(Renderable, BaseCommand):
 
     help = """
         Decrypt and rename all files in our collection into a given target
-        directory.  Note that we don't export any of the parsed data since
-        that can always be re-collected via the consumer.
+        directory.  And include a manifest file containing document data for
+        easy import.
     """.replace("    ", "")
 
     def add_arguments(self, parser):
         parser.add_argument("target")
 
     def __init__(self, *args, **kwargs):
-        self.verbosity = 0
-        self.target = None
         BaseCommand.__init__(self, *args, **kwargs)
+        self.target = None
 
     def handle(self, *args, **options):
 
-        self.verbosity = options["verbosity"]
         self.target = options["target"]
 
         if not os.path.exists(self.target):
@@ -40,9 +40,15 @@ class Command(Renderable, BaseCommand):
         if not settings.PASSPHRASE:
             settings.PASSPHRASE = input("Please enter the passphrase: ")
 
-        for document in Document.objects.all():
+        documents = Document.objects.all()
+        document_map = {d.pk: d for d in documents}
+        manifest = json.loads(serializers.serialize("json", documents))
+        for document_dict in manifest:
+
+            document = document_map[document_dict["pk"]]
 
             target = os.path.join(self.target, document.file_name)
+            document_dict["__exported_file_name__"] = target
 
             print("Exporting: {}".format(target))
 
@@ -50,3 +56,12 @@ class Command(Renderable, BaseCommand):
                 f.write(GnuPG.decrypted(document.source_file))
                 t = int(time.mktime(document.created.timetuple()))
                 os.utime(target, times=(t, t))
+
+        manifest += json.loads(
+            serializers.serialize("json", Correspondent.objects.all()))
+
+        manifest += json.loads(serializers.serialize(
+            "json", Tag.objects.all()))
+
+        with open(os.path.join(self.target, "manifest.json"), "w") as f:
+            json.dump(manifest, f, indent=2)

src/documents/management/commands/document_importer.py

@@ -0,0 +1,110 @@
+import json
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management import call_command
+
+from documents.models import Document
+from paperless.db import GnuPG
+
+from ...mixins import Renderable
+
+
+class Command(Renderable, BaseCommand):
+
+    help = """
+        Using a manifest.json file, load the data from there, and import the
+        documents it refers to.
+    """.replace("    ", "")
+
+    def add_arguments(self, parser):
+        parser.add_argument("source")
+        parser.add_argument(
+            '--ignore-absent',
+            action='store_true',
+            default=False,
+            help="If the manifest refers to a document that doesn't exist, "
+                 "ignore it and attempt to import what it can"
+        )
+
+    def __init__(self, *args, **kwargs):
+        BaseCommand.__init__(self, *args, **kwargs)
+        self.source = None
+        self.manifest = None
+
+    def handle(self, *args, **options):
+
+        self.source = options["source"]
+
+        if not os.path.exists(self.source):
+            raise CommandError("That path doesn't exist")
+
+        if not os.access(self.source, os.R_OK):
+            raise CommandError("That path doesn't appear to be readable")
+
+        manifest_path = os.path.join(self.source, "manifest.json")
+        self._check_manifest_exists(manifest_path)
+
+        with open(manifest_path) as f:
+            self.manifest = json.load(f)
+
+        self._check_manifest()
+
+        if not settings.PASSPHRASE:
+            raise CommandError(
+                "You need to define a passphrase before continuing.  Please "
+                "consult the documentation for setting up Paperless."
+            )
+
+        # Fill up the database with whatever is in the manifest
+        call_command("loaddata", manifest_path)
+
+        self._import_files_from_manifest()
+
+    @staticmethod
+    def _check_manifest_exists(path):
+        if not os.path.exists(path):
+            raise CommandError(
+                "That directory doesn't appear to contain a manifest.json "
+                "file."
+            )
+
+    def _check_manifest(self):
+
+        for record in self.manifest:
+
+            if not record["model"] == "documents.document":
+                continue
+
+            if "__exported_file_name__" not in record:
+                raise CommandError(
+                    'The manifest file contains a record which does not '
+                    'refer to an actual document file.  If you want to import '
+                    'the rest anyway (skipping such references) call the '
+                    'importer with --ignore-absent'
+                )
+
+            doc_file = record["__exported_file_name__"]
+            if not os.path.exists(os.path.join(self.source, doc_file)):
+                raise CommandError(
+                    'The manifest file refers to "{}" which does not '
+                    'appear to be in the source directory.  If you want to '
+                    'import the rest anyway (skipping such references) call '
+                    'the importer with --ignore-absent'.format(doc_file)
+                )
+
+    def _import_files_from_manifest(self):
+
+        for record in self.manifest:
+
+            if not record["model"] == "documents.document":
+                continue
+
+            doc_file = record["__exported_file_name__"]
+            document = Document.objects.get(pk=record["pk"])
+            with open(doc_file, "rb") as unencrypted:
+                with open(document.source_path, "wb") as encrypted:
+                    print("Encrypting {} and saving it to {}".format(
+                        doc_file, document.source_path))
+                    encrypted.write(GnuPG.encrypted(unencrypted))