Run importer

src/paperless_migration/settings.py

@@ -201,6 +201,7 @@ MIGRATION_TRANSFORMED_PATH = __get_path(
     "PAPERLESS_MIGRATION_TRANSFORMED_PATH",
     EXPORT_DIR / "manifest.v3.json",
 )
+MIGRATION_IMPORTED_PATH = Path(EXPORT_DIR / "import.completed").resolve()
 
 # One-time access code required for migration logins; stable across autoreload
 _code = os.getenv("PAPERLESS_MIGRATION_ACCESS_CODE")

src/paperless_migration/templates/paperless_migration/migration_home.html

@@ -81,7 +81,7 @@
         left: 0;
         top: 0;
         bottom: 0;
-        width: calc({{ export_exists|yesno:'33,0' }}% + {{ transformed_exists|yesno:'33,0' }}%);
+        width: calc({{ export_exists|yesno:'33,0' }}% + {{ transformed_exists|yesno:'33,0' }}% + {{ imported_exists|yesno:'34,0' }}%);
         max-width: 100%;
         background: linear-gradient(90deg, #17541f, #2c7a3c);
         border-radius: 999px;
@@ -143,7 +143,7 @@
                   </div>
                 </div>
                 <div class="d-flex align-items-center gap-2">
-                  <span class="step-chip {% if transformed_exists %}done{% endif %}">3</span>
+                  <span class="step-chip {% if imported_exists %}done{% endif %}">3</span>
                   <div>
                     <div class="fw-semibold mb-0">Import</div>
                     <small class="text-muted">into v3</small>
@@ -238,7 +238,7 @@
         </div>
 
         <div class="col-lg-3 col-md-4">
-          <div class="card card-step h-100 {% if transformed_exists %}done-step{% endif %}">
+          <div class="card card-step h-100 {% if imported_exists %}done-step{% endif %}">
             <div class="card-body d-flex flex-column gap-3">
               <div>
                 <p class="text-uppercase text-muted mb-1 fw-semibold" style="letter-spacing: 0.08rem;">Step 3</p>
@@ -253,7 +253,7 @@
                     type="submit"
                     name="action"
                     value="import"
-                    {% if not transformed_exists %}disabled aria-disabled="true"{% endif %}
+                    {% if not transformed_exists or imported_exists %}disabled aria-disabled="true"{% endif %}
                   >
                     Import transformed data
                   </button>
@@ -289,19 +289,21 @@
         </div>
       </div>
     </div>
-    {% if start_stream %}
+    {% if stream_action %}
       <script>
         (() => {
           const logEl = document.getElementById('migration-log');
           if (!logEl) return;
-          const evt = new EventSource('{% url "transform_stream" %}');
+          const streamUrl = "{% if stream_action == 'import' %}{% url 'import_stream' %}{% else %}{% url 'transform_stream' %}{% endif %}";
+          const donePrefix = "{{ stream_action|capfirst }} finished";
+          const evt = new EventSource(streamUrl);
           const append = (line) => {
             logEl.textContent += `\n${line}`;
             logEl.scrollTop = logEl.scrollHeight;
           };
           evt.onmessage = (e) => {
             append(e.data);
-            if (e.data.startsWith('Transform finished')) {
+            if (e.data.startsWith(donePrefix)) {
               setTimeout(() => window.location.reload(), 500);
             }
           };

src/paperless_migration/urls.py

@@ -11,6 +11,7 @@ urlpatterns = [
     path("accounts/", include("allauth.urls")),
     path("migration/", views.migration_home, name="migration_home"),
     path("migration/transform/stream", views.transform_stream, name="transform_stream"),
+    path("migration/import/stream", views.import_stream, name="import_stream"),
     # redirect root to migration home
     path("", views.migration_home, name="migration_home"),
 ]

src/paperless_migration/views.py

@@ -1,5 +1,8 @@
+import os
+import shutil
 import subprocess
 import sys
+import tempfile
 from pathlib import Path
 
 from django.contrib import messages
@@ -25,6 +28,7 @@ def migration_home(request):
 
     export_path = Path(settings.MIGRATION_EXPORT_PATH)
     transformed_path = Path(settings.MIGRATION_TRANSFORMED_PATH)
+    imported_marker = Path(settings.MIGRATION_IMPORTED_PATH)
 
     if request.method == "POST":
         action = request.POST.get("action")
@@ -32,7 +36,9 @@ def migration_home(request):
             messages.success(request, "Checked export paths.")
         elif action == "transform":
             messages.info(request, "Starting transform… live output below.")
-            request.session["start_transform_stream"] = True
+            request.session["start_stream_action"] = "transform"
+            if imported_marker.exists():
+                imported_marker.unlink()
         elif action == "upload":
             upload = request.FILES.get("export_file")
             if not upload:
@@ -47,20 +53,20 @@ def migration_home(request):
                 except Exception as exc:
                     messages.error(request, f"Failed to save file: {exc}")
         elif action == "import":
-            messages.info(
+            messages.info(request, "Starting import… live output below.")
-                request,
+            request.session["start_stream_action"] = "import"
-                "Import step is not implemented yet.",
-            )
         else:
             messages.error(request, "Unknown action.")
         return redirect("migration_home")
 
+    stream_action = request.session.pop("start_stream_action", None)
     context = {
         "export_path": export_path,
         "export_exists": export_path.exists(),
         "transformed_path": transformed_path,
         "transformed_exists": transformed_path.exists(),
-        "start_stream": request.session.pop("start_transform_stream", False),
+        "imported_exists": imported_marker.exists(),
+        "stream_action": stream_action,
     }
     return render(request, "paperless_migration/migration_home.html", context)
 
@@ -140,3 +146,89 @@ def transform_stream(request):
             "X-Accel-Buffering": "no",
         },
     )
+
+
+@login_required
+@require_http_methods(["GET"])
+def import_stream(request):
+    if not request.session.get("migration_code_ok"):
+        return HttpResponseForbidden("Access code required")
+    if not request.user.is_superuser:
+        return HttpResponseForbidden("Superuser access required")
+
+    export_path = Path(settings.MIGRATION_EXPORT_PATH)
+    transformed_path = Path(settings.MIGRATION_TRANSFORMED_PATH)
+    imported_marker = Path(settings.MIGRATION_IMPORTED_PATH)
+    manage_path = Path(settings.BASE_DIR) / "manage.py"
+    source_dir = export_path.parent
+
+    cmd = [
+        sys.executable,
+        str(manage_path),
+        "document_importer",
+        str(source_dir),
+        "--data-only",
+    ]
+
+    env = os.environ.copy()
+    env["DJANGO_SETTINGS_MODULE"] = "paperless.settings"
+    env["PAPERLESS_MIGRATION_MODE"] = "0"
+
+    def event_stream():
+        if not export_path.exists():
+            yield "data: Missing export manifest.json; upload or re-check export.\n\n"
+            return
+        if not transformed_path.exists():
+            yield "data: Missing transformed manifest.v3.json; run transform first.\n\n"
+            return
+
+        backup_path: Path | None = None
+        try:
+            backup_fd, backup_name = tempfile.mkstemp(
+                prefix="manifest.v2.",
+                suffix=".json",
+                dir=source_dir,
+            )
+            os.close(backup_fd)
+            backup_path = Path(backup_name)
+            shutil.copy2(export_path, backup_path)
+            shutil.copy2(transformed_path, export_path)
+        except Exception as exc:
+            yield f"data: Failed to prepare import manifest: {exc}\n\n"
+            return
+
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            bufsize=1,
+            text=True,
+            env=env,
+        )
+        try:
+            yield "data: Starting import...\n\n"
+            if process.stdout:
+                for line in process.stdout:
+                    yield f"data: {line.rstrip()}\n\n"
+            process.wait()
+            if process.returncode == 0:
+                imported_marker.parent.mkdir(parents=True, exist_ok=True)
+                imported_marker.write_text("ok\n", encoding="utf-8")
+            yield f"data: Import finished with code {process.returncode}\n\n"
+        finally:
+            if process and process.poll() is None:
+                process.kill()
+            if backup_path and backup_path.exists():
+                try:
+                    shutil.move(backup_path, export_path)
+                except Exception:
+                    pass
+
+    return StreamingHttpResponse(
+        event_stream(),
+        content_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",
+        },
+    )