Feature: Allow encrypting sensitive fields in export (#6927)

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

src/documents/tests/test_management_exporter.py

@@ -3,6 +3,7 @@ import json
 import os
 import shutil
 import tempfile
+from io import StringIO
 from pathlib import Path
 from unittest import mock
 from zipfile import ZipFile
@@ -39,6 +40,7 @@ from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import FileSystemAssertsMixin
 from documents.tests.utils import SampleDirMixin
 from documents.tests.utils import paperless_environment
+from paperless_mail.models import MailAccount
 
 
 class TestExportImport(
@@ -466,7 +468,7 @@ class TestExportImport(
         with ZipFile(expected_file) as zip:
             self.assertEqual(len(zip.namelist()), 11)
             self.assertIn("manifest.json", zip.namelist())
-            self.assertIn("version.json", zip.namelist())
+            self.assertIn("metadata.json", zip.namelist())
 
     @override_settings(PASSPHRASE="test")
     def test_export_zipped_format(self):
@@ -504,7 +506,7 @@ class TestExportImport(
             # Extras are from the directories, which also appear in the listing
             self.assertEqual(len(zip.namelist()), 14)
             self.assertIn("manifest.json", zip.namelist())
-            self.assertIn("version.json", zip.namelist())
+            self.assertIn("metadata.json", zip.namelist())
 
     @override_settings(PASSPHRASE="test")
     def test_export_zipped_with_delete(self):
@@ -552,7 +554,7 @@ class TestExportImport(
         with ZipFile(expected_file) as zip:
             self.assertEqual(len(zip.namelist()), 11)
             self.assertIn("manifest.json", zip.namelist())
-            self.assertIn("version.json", zip.namelist())
+            self.assertIn("metadata.json", zip.namelist())
 
     def test_export_target_not_exists(self):
         """
@@ -827,7 +829,7 @@ class TestExportImport(
         # Manifest and version files only should be present in the exported directory
         self.assertFileCountInDir(self.target, 2)
         self.assertIsFile(self.target / "manifest.json")
-        self.assertIsFile(self.target / "version.json")
+        self.assertIsFile(self.target / "metadata.json")
 
         shutil.rmtree(self.dirs.media_dir / "documents")
         Document.objects.all().delete()
@@ -840,3 +842,139 @@ class TestExportImport(
         )
 
         self.assertEqual(Document.objects.all().count(), 4)
+
+
+class TestCryptExportImport(
+    DirectoriesMixin,
+    FileSystemAssertsMixin,
+    TestCase,
+):
+    def setUp(self) -> None:
+        self.target = Path(tempfile.mkdtemp())
+        return super().setUp()
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.target, ignore_errors=True)
+        return super().tearDown()
+
+    def test_export_passphrase(self):
+        """
+        GIVEN:
+            - A mail account exists
+        WHEN:
+            - Export command is called
+            - Passphrase is provided
+        THEN:
+            - Output password is not plaintext
+        """
+        MailAccount.objects.create(
+            name="Test Account",
+            imap_server="test.imap.com",
+            username="myusername",
+            password="mypassword",
+        )
+
+        call_command(
+            "document_exporter",
+            "--no-progress-bar",
+            "--passphrase",
+            "securepassword",
+            self.target,
+        )
+
+        self.assertIsFile(self.target / "metadata.json")
+        self.assertIsFile(self.target / "manifest.json")
+
+        data = json.loads((self.target / "manifest.json").read_text())
+
+        mail_accounts = list(
+            filter(lambda r: r["model"] == "paperless_mail.mailaccount", data),
+        )
+
+        self.assertEqual(len(mail_accounts), 1)
+
+        mail_account_data = mail_accounts[0]
+
+        self.assertNotEqual(mail_account_data["fields"]["password"], "mypassword")
+
+        MailAccount.objects.all().delete()
+
+        call_command(
+            "document_importer",
+            "--no-progress-bar",
+            "--passphrase",
+            "securepassword",
+            self.target,
+        )
+
+        account = MailAccount.objects.first()
+
+        self.assertIsNotNone(account)
+        self.assertEqual(account.password, "mypassword")
+
+    def test_import_crypt_no_passphrase(self):
+        """
+        GIVEN:
+            - A mail account exists
+        WHEN:
+            - Export command is called
+            - Passphrase is provided
+            - Import command is called
+            - No passphrase is given
+        THEN:
+            - An error is raised for the issue
+        """
+        call_command(
+            "document_exporter",
+            "--no-progress-bar",
+            "--passphrase",
+            "securepassword",
+            self.target,
+        )
+
+        with self.assertRaises(CommandError) as err:
+            call_command(
+                "document_importer",
+                "--no-progress-bar",
+                self.target,
+            )
+            self.assertEqual(
+                err.msg,
+                "No passphrase was given, but this export contains encrypted fields",
+            )
+
+    def test_export_warn_plaintext(self):
+        """
+        GIVEN:
+            - A mail account exists
+        WHEN:
+            - Export command is called
+            - No passphrase is provided
+        THEN:
+            - Output password is plaintext
+            - Warning is output
+        """
+        MailAccount.objects.create(
+            name="Test Account",
+            imap_server="test.imap.com",
+            username="myusername",
+            password="mypassword",
+        )
+
+        stdout = StringIO()
+
+        call_command(
+            "document_exporter",
+            "--no-progress-bar",
+            str(self.target),
+            stdout=stdout,
+        )
+        stdout.seek(0)
+        self.assertIn(
+            (
+                "You have configured mail accounts, "
+                "but no passphrase was given. "
+                "Passwords will be in plaintext"
+            ),
+            stdout.read(),
+        )

src/documents/tests/test_management_importer.py

@@ -125,15 +125,16 @@ class TestCommandImport(
                     EXPORTER_ARCHIVE_NAME: "archive.pdf",
                 },
             ]
+            cmd.data_only = False
             with self.assertRaises(CommandError) as cm:
-                cmd._check_manifest_files_valid()
+                cmd.check_manifest_validity()
                 self.assertInt("Failed to read from original file", str(cm.exception))
 
             original_path.chmod(0o444)
             archive_path.chmod(0o222)
 
             with self.assertRaises(CommandError) as cm:
-                cmd._check_manifest_files_valid()
+                cmd.check_manifest_validity()
                 self.assertInt("Failed to read from archive file", str(cm.exception))
 
     def test_import_source_not_existing(self):
@@ -240,7 +241,7 @@ class TestCommandImport(
         stdout.seek(0)
         self.assertIn(
             "Found existing user(s), this might indicate a non-empty installation",
-            str(stdout.read()),
+            stdout.read(),
         )
 
     def test_import_with_documents_exists(self):
@@ -278,3 +279,59 @@ class TestCommandImport(
             "Found existing documents(s), this might indicate a non-empty installation",
             str(stdout.read()),
         )
+
+    def test_import_no_metadata_or_version_file(self):
+        """
+        GIVEN:
+            - A source directory with a manifest file only
+        WHEN:
+            - An import is attempted
+        THEN:
+            - Warning about the missing files is output
+        """
+        stdout = StringIO()
+
+        (self.dirs.scratch_dir / "manifest.json").touch()
+
+        # We're not building a manifest, so it fails, but this test doesn't care
+        with self.assertRaises(json.decoder.JSONDecodeError):
+            call_command(
+                "document_importer",
+                "--no-progress-bar",
+                str(self.dirs.scratch_dir),
+                stdout=stdout,
+            )
+        stdout.seek(0)
+        stdout_str = str(stdout.read())
+
+        self.assertIn("No version.json or metadata.json file located", stdout_str)
+
+    def test_import_version_file(self):
+        """
+        GIVEN:
+            - A source directory with a manifest file and version file
+        WHEN:
+            - An import is attempted
+        THEN:
+            - Warning about the the version mismatch is output
+        """
+        stdout = StringIO()
+
+        (self.dirs.scratch_dir / "manifest.json").touch()
+        (self.dirs.scratch_dir / "version.json").write_text(
+            json.dumps({"version": "2.8.1"}),
+        )
+
+        # We're not building a manifest, so it fails, but this test doesn't care
+        with self.assertRaises(json.decoder.JSONDecodeError):
+            call_command(
+                "document_importer",
+                "--no-progress-bar",
+                str(self.dirs.scratch_dir),
+                stdout=stdout,
+            )
+        stdout.seek(0)
+        stdout_str = str(stdout.read())
+
+        self.assertIn("Version mismatch:", stdout_str)
+        self.assertIn("importing 2.8.1", stdout_str)