mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-01-26 22:49:01 -06:00
Compare commits
3 Commits
feature/mi
...
feature-bu
| Author | SHA1 | Date | |
|---|---|---|---|
|
0df0f3a21f | ||
|
|
d81488d054 | ||
|
|
8dc4f34743 |
1
.github/release-drafter.yml
vendored
1
.github/release-drafter.yml
vendored
@@ -44,7 +44,6 @@ include-labels:
|
||||
- 'notable'
|
||||
exclude-labels:
|
||||
- 'skip-changelog'
|
||||
filter-by-commitish: true
|
||||
category-template: '### $TITLE'
|
||||
change-template: '- $TITLE @$AUTHOR ([#$NUMBER]($URL))'
|
||||
change-title-escapes: '\<*_&#@'
|
||||
|
||||
@@ -37,7 +37,7 @@ repos:
|
||||
- json
|
||||
# See https://github.com/prettier/prettier/issues/15742 for the fork reason
|
||||
- repo: https://github.com/rbubley/mirrors-prettier
|
||||
rev: 'v3.6.2'
|
||||
rev: 'v3.8.0'
|
||||
hooks:
|
||||
- id: prettier
|
||||
types_or:
|
||||
@@ -49,7 +49,7 @@ repos:
|
||||
- 'prettier-plugin-organize-imports@4.1.0'
|
||||
# Python hooks
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.14.5
|
||||
rev: v0.14.13
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
- id: ruff-format
|
||||
@@ -76,7 +76,7 @@ repos:
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
- repo: https://github.com/google/yamlfmt
|
||||
rev: v0.20.0
|
||||
rev: v0.21.0
|
||||
hooks:
|
||||
- id: yamlfmt
|
||||
exclude: "^src-ui/pnpm-lock.yaml"
|
||||
|
||||
@@ -8,11 +8,6 @@ echo "${log_prefix} Apply database migrations..."
|
||||
|
||||
cd "${PAPERLESS_SRC_DIR}"
|
||||
|
||||
if [[ "${PAPERLESS_MIGRATION_MODE:-0}" == "1" ]]; then
|
||||
echo "${log_prefix} Migration mode enabled, skipping migrations."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The whole migrate, with flock, needs to run as the right user
|
||||
if [[ -n "${USER_IS_NON_ROOT}" ]]; then
|
||||
exec s6-setlock -n "${data_dir}/migration_lock" python3 manage.py migrate --skip-checks --no-input
|
||||
|
||||
@@ -9,15 +9,7 @@ echo "${log_prefix} Running Django checks"
|
||||
cd "${PAPERLESS_SRC_DIR}"
|
||||
|
||||
if [[ -n "${USER_IS_NON_ROOT}" ]]; then
|
||||
if [[ "${PAPERLESS_MIGRATION_MODE:-0}" == "1" ]]; then
|
||||
python3 manage_migration.py check
|
||||
else
|
||||
python3 manage.py check
|
||||
fi
|
||||
else
|
||||
if [[ "${PAPERLESS_MIGRATION_MODE:-0}" == "1" ]]; then
|
||||
s6-setuidgid paperless python3 manage_migration.py check
|
||||
else
|
||||
s6-setuidgid paperless python3 manage.py check
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -13,14 +13,8 @@ if [[ -n "${PAPERLESS_FORCE_SCRIPT_NAME}" ]]; then
|
||||
export GRANIAN_URL_PATH_PREFIX=${PAPERLESS_FORCE_SCRIPT_NAME}
|
||||
fi
|
||||
|
||||
if [[ "${PAPERLESS_MIGRATION_MODE:-0}" == "1" ]]; then
|
||||
app_module="paperless.migration_asgi:application"
|
||||
else
|
||||
app_module="paperless.asgi:application"
|
||||
fi
|
||||
|
||||
if [[ -n "${USER_IS_NON_ROOT}" ]]; then
|
||||
exec granian --interface asginl --ws --loop uvloop "${app_module}"
|
||||
exec granian --interface asginl --ws --loop uvloop "paperless.asgi:application"
|
||||
else
|
||||
exec s6-setuidgid paperless granian --interface asginl --ws --loop uvloop "${app_module}"
|
||||
exec s6-setuidgid paperless granian --interface asginl --ws --loop uvloop "paperless.asgi:application"
|
||||
fi
|
||||
|
||||
@@ -1,21 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## paperless-ngx 2.20.5
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix: ensure horizontal scroll for long tag names in list, wrap tags without parent [@shamoon](https://github.com/shamoon) ([#11811](https://github.com/paperless-ngx/paperless-ngx/pull/11811))
|
||||
- Fix: use explicit order field for workflow actions [@shamoon](https://github.com/shamoon) [@stumpylog](https://github.com/stumpylog) ([#11781](https://github.com/paperless-ngx/paperless-ngx/pull/11781))
|
||||
|
||||
### All App Changes
|
||||
|
||||
<details>
|
||||
<summary>2 changes</summary>
|
||||
|
||||
- Fix: ensure horizontal scroll for long tag names in list, wrap tags without parent [@shamoon](https://github.com/shamoon) ([#11811](https://github.com/paperless-ngx/paperless-ngx/pull/11811))
|
||||
- Fix: use explicit order field for workflow actions [@shamoon](https://github.com/shamoon) [@stumpylog](https://github.com/stumpylog) ([#11781](https://github.com/paperless-ngx/paperless-ngx/pull/11781))
|
||||
</details>
|
||||
|
||||
## paperless-ngx 2.20.4
|
||||
|
||||
### Security
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "paperless-ngx"
|
||||
version = "2.20.5"
|
||||
version = "2.20.4"
|
||||
description = "A community-supported supercharged document management system: scan, index and archive all your physical documents"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
@@ -19,15 +19,15 @@ dependencies = [
|
||||
"azure-ai-documentintelligence>=1.0.2",
|
||||
"babel>=2.17",
|
||||
"bleach~=6.3.0",
|
||||
"celery[redis]~=5.5.1",
|
||||
"celery[redis]~=5.6.2",
|
||||
"channels~=4.2",
|
||||
"channels-redis~=4.2",
|
||||
"concurrent-log-handler~=0.9.25",
|
||||
"dateparser~=1.2",
|
||||
# WARNING: django does not use semver.
|
||||
# Only patch versions are guaranteed to not introduce breaking changes.
|
||||
"django~=5.2.5",
|
||||
"django-allauth[mfa,socialaccount]~=65.12.1",
|
||||
"django==5.2.10",
|
||||
"django-allauth[mfa,socialaccount]~=65.13.0",
|
||||
"django-auditlog~=3.4.1",
|
||||
"django-cachalot~=2.8.0",
|
||||
"django-celery-results~=2.6.0",
|
||||
@@ -49,7 +49,6 @@ dependencies = [
|
||||
"flower~=2.0.1",
|
||||
"gotenberg-client~=0.13.1",
|
||||
"httpx-oauth~=0.16",
|
||||
"ijson~=3.3",
|
||||
"imap-tools~=1.11.0",
|
||||
"inotifyrecursive~=0.3",
|
||||
"jinja2~=3.1.5",
|
||||
@@ -74,16 +73,14 @@ dependencies = [
|
||||
"rapidfuzz~=3.14.0",
|
||||
"redis[hiredis]~=5.2.1",
|
||||
"regex>=2025.9.18",
|
||||
"rich~=14.1.0",
|
||||
"scikit-learn~=1.7.0",
|
||||
"sentence-transformers>=4.1",
|
||||
"setproctitle~=1.3.4",
|
||||
"tika-client~=0.10.0",
|
||||
"torch~=2.9.1",
|
||||
"tqdm~=4.67.1",
|
||||
"typer~=0.12",
|
||||
"watchdog~=6.0",
|
||||
"whitenoise~=6.9",
|
||||
"whitenoise~=6.11",
|
||||
"whoosh-reloaded>=2.7.5",
|
||||
"zxing-cpp~=2.3.0",
|
||||
]
|
||||
@@ -92,13 +89,13 @@ optional-dependencies.mariadb = [
|
||||
"mysqlclient~=2.2.7",
|
||||
]
|
||||
optional-dependencies.postgres = [
|
||||
"psycopg[c,pool]==3.2.12",
|
||||
"psycopg[c,pool]==3.3",
|
||||
# Direct dependency for proper resolution of the pre-built wheels
|
||||
"psycopg-c==3.2.12",
|
||||
"psycopg-c==3.3",
|
||||
"psycopg-pool==3.3",
|
||||
]
|
||||
optional-dependencies.webserver = [
|
||||
"granian[uvloop]~=2.5.1",
|
||||
"granian[uvloop]~=2.6.0",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
@@ -155,7 +152,7 @@ typing = [
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
required-version = ">=0.5.14"
|
||||
required-version = ">=0.9.0"
|
||||
package = false
|
||||
environments = [
|
||||
"sys_platform == 'darwin'",
|
||||
@@ -165,8 +162,8 @@ environments = [
|
||||
[tool.uv.sources]
|
||||
# Markers are chosen to select these almost exclusively when building the Docker image
|
||||
psycopg-c = [
|
||||
{ url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-bookworm-3.2.12/psycopg_c-3.2.12-cp312-cp312-linux_x86_64.whl", marker = "sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.12'" },
|
||||
{ url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-bookworm-3.2.12/psycopg_c-3.2.12-cp312-cp312-linux_aarch64.whl", marker = "sys_platform == 'linux' and platform_machine == 'aarch64' and python_version == '3.12'" },
|
||||
{ url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.0/psycopg_c-3.3.0-cp312-cp312-linux_x86_64.whl", marker = "sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.12'" },
|
||||
{ url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.0/psycopg_c-3.3.0-cp312-cp312-linux_aarch64.whl", marker = "sys_platform == 'linux' and platform_machine == 'aarch64' and python_version == '3.12'" },
|
||||
]
|
||||
zxing-cpp = [
|
||||
{ url = "https://github.com/paperless-ngx/builder/releases/download/zxing-2.3.0/zxing_cpp-2.3.0-cp312-cp312-linux_x86_64.whl", marker = "sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.12'" },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "paperless-ngx-ui",
|
||||
"version": "2.20.5",
|
||||
"version": "2.20.4",
|
||||
"scripts": {
|
||||
"preinstall": "npx only-allow pnpm",
|
||||
"ng": "ng",
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
</button>
|
||||
</ng-template>
|
||||
<ng-template ng-option-tmp let-item="item" let-index="index" let-search="searchTerm">
|
||||
<div class="tag-option-row d-flex align-items-center" [class.w-auto]="!getTag(item.id)?.parent">
|
||||
<div class="tag-option-row d-flex align-items-center">
|
||||
@if (item.id && tags) {
|
||||
@if (getTag(item.id)?.parent) {
|
||||
<i-bs name="list-nested" class="me-1"></i-bs>
|
||||
|
||||
@@ -22,8 +22,8 @@
|
||||
}
|
||||
|
||||
// Dropdown hierarchy reveal for ng-select options
|
||||
:host ::ng-deep .ng-dropdown-panel .ng-option {
|
||||
overflow-x: auto !important;
|
||||
::ng-deep .ng-dropdown-panel .ng-option {
|
||||
overflow-x: scroll;
|
||||
|
||||
.tag-option-row {
|
||||
font-size: 1rem;
|
||||
@@ -41,12 +41,12 @@
|
||||
}
|
||||
}
|
||||
|
||||
:host ::ng-deep .ng-dropdown-panel .ng-option:hover .hierarchy-reveal,
|
||||
:host ::ng-deep .ng-dropdown-panel .ng-option.ng-option-marked .hierarchy-reveal {
|
||||
::ng-deep .ng-dropdown-panel .ng-option:hover .hierarchy-reveal,
|
||||
::ng-deep .ng-dropdown-panel .ng-option.ng-option-marked .hierarchy-reveal {
|
||||
max-width: 1000px;
|
||||
}
|
||||
|
||||
::ng-deep .ng-dropdown-panel .ng-option:hover .hierarchy-indicator,
|
||||
:host ::ng-deep .ng-dropdown-panel .ng-option.ng-option-marked .hierarchy-indicator {
|
||||
::ng-deep .ng-dropdown-panel .ng-option.ng-option-marked .hierarchy-indicator {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
@@ -285,10 +285,10 @@ export class DocumentDetailComponent
|
||||
if (
|
||||
element &&
|
||||
element.nativeElement.offsetParent !== null &&
|
||||
this.nav?.activeId == DocumentDetailNavIDs.Preview
|
||||
this.nav?.activeId == 4
|
||||
) {
|
||||
// its visible
|
||||
setTimeout(() => this.nav?.select(DocumentDetailNavIDs.Details))
|
||||
setTimeout(() => this.nav?.select(1))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ export const environment = {
|
||||
apiVersion: '9', // match src/paperless/settings.py
|
||||
appTitle: 'Paperless-ngx',
|
||||
tag: 'prod',
|
||||
version: '2.20.5',
|
||||
version: '2.20.4',
|
||||
webSocketHost: window.location.host,
|
||||
webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
|
||||
webSocketBaseUrl: base_url.pathname + 'ws/',
|
||||
|
||||
@@ -6,7 +6,7 @@ from django.db import models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "1075_workflowaction_order"),
|
||||
("documents", "1074_workflowrun_deleted_at_workflowrun_restored_at_and_more"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -12,7 +12,7 @@ def populate_action_order(apps, schema_editor):
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "1074_workflowrun_deleted_at_workflowrun_restored_at_and_more"),
|
||||
("documents", "1075_alter_paperlesstask_task_name"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -3,11 +3,6 @@ import os
|
||||
import sys
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
from paperless_migration.detect import choose_settings_module
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", choose_settings_module())
|
||||
except Exception:
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
||||
|
||||
from django.core.management import execute_from_command_line
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.environ.setdefault(
|
||||
"DJANGO_SETTINGS_MODULE",
|
||||
"paperless_migration.settings",
|
||||
)
|
||||
|
||||
from django.core.management import execute_from_command_line
|
||||
|
||||
execute_from_command_line(sys.argv)
|
||||
@@ -1,18 +1,12 @@
|
||||
import os
|
||||
|
||||
try:
|
||||
from paperless_migration.detect import choose_settings_module
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", choose_settings_module())
|
||||
except Exception:
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
||||
|
||||
from django.core.asgi import get_asgi_application
|
||||
|
||||
# Fetch Django ASGI application early to ensure AppRegistry is populated
|
||||
# before importing consumers and AuthMiddlewareStack that may import ORM
|
||||
# models.
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
||||
django_asgi_app = get_asgi_application()
|
||||
|
||||
from channels.auth import AuthMiddlewareStack # noqa: E402
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
import os
|
||||
|
||||
from django.core.asgi import get_asgi_application
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless_migration.settings")
|
||||
|
||||
application = get_asgi_application()
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import Final
|
||||
|
||||
__version__: Final[tuple[int, int, int]] = (2, 20, 5)
|
||||
__version__: Final[tuple[int, int, int]] = (2, 20, 4)
|
||||
# Version string like X.Y.Z
|
||||
__full_version_str__: Final[str] = ".".join(map(str, __version__))
|
||||
# Version string like X.Y
|
||||
|
||||
@@ -9,15 +9,10 @@ https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/
|
||||
|
||||
import os
|
||||
|
||||
try:
|
||||
from paperless_migration.detect import choose_settings_module
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", choose_settings_module())
|
||||
except Exception:
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
||||
|
||||
from django.core.wsgi import get_wsgi_application
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
||||
|
||||
application = get_wsgi_application()
|
||||
|
||||
import logging # noqa: E402
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class PaperlessMigrationConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "paperless_migration"
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Lightweight detection to decide if we should boot migration mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
_DOC_EXISTS_QUERY = "SELECT 1 FROM documents_document LIMIT 1;"
|
||||
|
||||
|
||||
def _get_db_config() -> dict[str, Any]:
|
||||
data_dir = Path(os.getenv("PAPERLESS_DATA_DIR", BASE_DIR.parent / "data")).resolve()
|
||||
if not os.getenv("PAPERLESS_DBHOST"):
|
||||
return {
|
||||
"ENGINE": "sqlite",
|
||||
"NAME": data_dir / "db.sqlite3",
|
||||
}
|
||||
|
||||
engine = "mariadb" if os.getenv("PAPERLESS_DBENGINE") == "mariadb" else "postgres"
|
||||
cfg = {
|
||||
"ENGINE": engine,
|
||||
"HOST": os.getenv("PAPERLESS_DBHOST"),
|
||||
"PORT": os.getenv("PAPERLESS_DBPORT"),
|
||||
"NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
|
||||
"USER": os.getenv("PAPERLESS_DBUSER", "paperless"),
|
||||
"PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"),
|
||||
}
|
||||
return cfg
|
||||
|
||||
|
||||
def _probe_sqlite(path: Path) -> bool:
|
||||
if not path.exists():
|
||||
return False
|
||||
try:
|
||||
conn = sqlite3.connect(path, timeout=1)
|
||||
cur = conn.cursor()
|
||||
cur.execute(_DOC_EXISTS_QUERY)
|
||||
cur.fetchone()
|
||||
return True
|
||||
except sqlite3.Error:
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _probe_postgres(cfg: dict[str, Any]) -> bool:
|
||||
try:
|
||||
import psycopg
|
||||
except ImportError: # pragma: no cover
|
||||
logger.debug("psycopg not installed; skipping postgres probe")
|
||||
return False
|
||||
|
||||
try:
|
||||
conn = psycopg.connect(
|
||||
host=cfg["HOST"],
|
||||
port=cfg["PORT"],
|
||||
dbname=cfg["NAME"],
|
||||
user=cfg["USER"],
|
||||
password=cfg["PASSWORD"],
|
||||
connect_timeout=2,
|
||||
)
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(_DOC_EXISTS_QUERY)
|
||||
cur.fetchone()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _probe_mariadb(cfg: dict[str, Any]) -> bool:
|
||||
try:
|
||||
import MySQLdb # type: ignore
|
||||
except ImportError: # pragma: no cover
|
||||
logger.debug("mysqlclient not installed; skipping mariadb probe")
|
||||
return False
|
||||
|
||||
try:
|
||||
conn = MySQLdb.connect(
|
||||
host=cfg["HOST"],
|
||||
port=int(cfg["PORT"] or 3306),
|
||||
user=cfg["USER"],
|
||||
passwd=cfg["PASSWORD"],
|
||||
db=cfg["NAME"],
|
||||
connect_timeout=2,
|
||||
)
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT 1 FROM documents_document LIMIT 1;")
|
||||
cur.fetchone()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def is_v2_database() -> bool:
|
||||
cfg = _get_db_config()
|
||||
if cfg["ENGINE"] == "sqlite":
|
||||
return _probe_sqlite(cfg["NAME"])
|
||||
if cfg["ENGINE"] == "postgres":
|
||||
return _probe_postgres(cfg)
|
||||
if cfg["ENGINE"] == "mariadb":
|
||||
return _probe_mariadb(cfg)
|
||||
return False
|
||||
|
||||
|
||||
def choose_settings_module() -> str:
|
||||
# ENV override
|
||||
toggle = os.getenv("PAPERLESS_MIGRATION_MODE")
|
||||
if toggle is not None:
|
||||
chosen = (
|
||||
"paperless_migration.settings"
|
||||
if str(toggle).lower() in ("1", "true", "yes", "on")
|
||||
else "paperless.settings"
|
||||
)
|
||||
os.environ["PAPERLESS_MIGRATION_MODE"] = "1" if "migration" in chosen else "0"
|
||||
return chosen
|
||||
|
||||
# Auto-detect via DB probe
|
||||
if is_v2_database():
|
||||
logger.warning("Detected v2 schema; booting migration mode.")
|
||||
os.environ["PAPERLESS_MIGRATION_MODE"] = "1"
|
||||
return "paperless_migration.settings"
|
||||
|
||||
os.environ["PAPERLESS_MIGRATION_MODE"] = "0"
|
||||
return "paperless.settings"
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
logger.info(
|
||||
"v2 database detected" if is_v2_database() else "v2 database not detected",
|
||||
)
|
||||
@@ -1,158 +0,0 @@
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "rich",
|
||||
# "ijson",
|
||||
# "typer-slim",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
import json
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import TypedDict
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.progress import BarColumn
|
||||
from rich.progress import Progress
|
||||
from rich.progress import SpinnerColumn
|
||||
from rich.progress import TextColumn
|
||||
from rich.progress import TimeElapsedColumn
|
||||
from rich.table import Table
|
||||
|
||||
try:
|
||||
import ijson # type: ignore
|
||||
except ImportError as exc: # pragma: no cover - handled at runtime
|
||||
raise SystemExit(
|
||||
"ijson is required for migration transform. "
|
||||
"Install dependencies (e.g., `uv pip install ijson`).",
|
||||
) from exc
|
||||
|
||||
app = typer.Typer(add_completion=False)
|
||||
console = Console()
|
||||
|
||||
|
||||
class FixtureObject(TypedDict):
|
||||
model: str
|
||||
pk: int
|
||||
fields: dict[str, Any]
|
||||
|
||||
|
||||
TransformFn = Callable[[FixtureObject], FixtureObject]
|
||||
|
||||
|
||||
def transform_documents_document(obj: FixtureObject) -> FixtureObject:
|
||||
fields: dict[str, Any] = obj["fields"]
|
||||
fields.pop("storage_type", None)
|
||||
content: Any = fields.get("content")
|
||||
fields["content_length"] = len(content) if isinstance(content, str) else 0
|
||||
return obj
|
||||
|
||||
|
||||
TRANSFORMS: dict[str, TransformFn] = {
|
||||
"documents.document": transform_documents_document,
|
||||
}
|
||||
|
||||
|
||||
def validate_output(value: Path) -> Path:
|
||||
if value.exists():
|
||||
raise typer.BadParameter(f"Output file '{value}' already exists.")
|
||||
return value
|
||||
|
||||
|
||||
@app.command()
|
||||
def migrate(
|
||||
input_path: Path = typer.Option(
|
||||
...,
|
||||
"--input",
|
||||
"-i",
|
||||
exists=True,
|
||||
file_okay=True,
|
||||
dir_okay=False,
|
||||
readable=True,
|
||||
),
|
||||
output_path: Path = typer.Option(
|
||||
...,
|
||||
"--output",
|
||||
"-o",
|
||||
callback=validate_output,
|
||||
),
|
||||
) -> None:
|
||||
"""
|
||||
Process JSON fixtures with detailed summary and timing.
|
||||
"""
|
||||
if input_path.resolve() == output_path.resolve():
|
||||
console.print(
|
||||
"[bold red]Error:[/bold red] Input and output paths cannot be the same file.",
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
stats: Counter[str] = Counter()
|
||||
total_processed: int = 0
|
||||
start_time: float = time.perf_counter()
|
||||
|
||||
progress = Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[bold blue]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("{task.completed:,} rows"),
|
||||
TimeElapsedColumn(),
|
||||
console=console,
|
||||
)
|
||||
|
||||
with (
|
||||
progress,
|
||||
input_path.open("rb") as infile,
|
||||
output_path.open("w", encoding="utf-8") as outfile,
|
||||
):
|
||||
task = progress.add_task("Processing fixture", start=True)
|
||||
outfile.write("[\n")
|
||||
first: bool = True
|
||||
|
||||
for i, obj in enumerate(ijson.items(infile, "item")):
|
||||
fixture: FixtureObject = obj
|
||||
model: str = fixture["model"]
|
||||
total_processed += 1
|
||||
|
||||
transform: TransformFn | None = TRANSFORMS.get(model)
|
||||
if transform:
|
||||
fixture = transform(fixture)
|
||||
stats[model] += 1
|
||||
|
||||
if not first:
|
||||
outfile.write(",\n")
|
||||
first = False
|
||||
|
||||
json.dump(fixture, outfile, ensure_ascii=False)
|
||||
progress.advance(task, 1)
|
||||
|
||||
outfile.write("\n]\n")
|
||||
|
||||
end_time: float = time.perf_counter()
|
||||
duration: float = end_time - start_time
|
||||
|
||||
# Final Statistics Table
|
||||
console.print("\n[bold green]Processing Complete[/bold green]")
|
||||
|
||||
table = Table(show_header=True, header_style="bold magenta")
|
||||
table.add_column("Metric", style="dim")
|
||||
table.add_column("Value", justify="right")
|
||||
|
||||
table.add_row("Total Time", f"{duration:.2f} seconds")
|
||||
table.add_row("Total Processed", f"{total_processed:,} rows")
|
||||
table.add_row(
|
||||
"Processing Speed",
|
||||
f"{total_processed / duration:.0f} rows/sec" if duration > 0 else "N/A",
|
||||
)
|
||||
|
||||
for model, count in stats.items():
|
||||
table.add_row(f"Transformed: {model}", f"{count:,}")
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
@@ -1,61 +0,0 @@
|
||||
import django
|
||||
from django.apps import apps
|
||||
from django.db import connection
|
||||
from django.db.migrations.recorder import MigrationRecorder
|
||||
|
||||
|
||||
def _target_tables() -> list[str]:
|
||||
tables = {
|
||||
model._meta.db_table for model in apps.get_models(include_auto_created=True)
|
||||
}
|
||||
tables.add(MigrationRecorder.Migration._meta.db_table)
|
||||
existing = set(connection.introspection.table_names())
|
||||
return sorted(tables & existing)
|
||||
|
||||
|
||||
def _drop_sqlite_tables() -> None:
|
||||
tables = _target_tables()
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("PRAGMA foreign_keys=OFF;")
|
||||
for table in tables:
|
||||
cursor.execute(f'DROP TABLE IF EXISTS "{table}";')
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
|
||||
|
||||
def _drop_postgres_tables() -> None:
|
||||
tables = _target_tables()
|
||||
if not tables:
|
||||
return
|
||||
with connection.cursor() as cursor:
|
||||
for table in tables:
|
||||
cursor.execute(f'DROP TABLE IF EXISTS "{table}" CASCADE;')
|
||||
|
||||
|
||||
def _drop_mysql_tables() -> None:
|
||||
tables = _target_tables()
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SET FOREIGN_KEY_CHECKS=0;")
|
||||
for table in tables:
|
||||
cursor.execute(f"DROP TABLE IF EXISTS `{table}`;")
|
||||
cursor.execute("SET FOREIGN_KEY_CHECKS=1;")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
django.setup()
|
||||
vendor = connection.vendor
|
||||
print(f"Wiping database for {vendor}...") # noqa: T201
|
||||
|
||||
if vendor == "sqlite":
|
||||
_drop_sqlite_tables()
|
||||
elif vendor == "postgresql":
|
||||
_drop_postgres_tables()
|
||||
elif vendor == "mysql":
|
||||
_drop_mysql_tables()
|
||||
else:
|
||||
raise SystemExit(f"Unsupported database vendor: {vendor}")
|
||||
|
||||
print("Database wipe complete.") # noqa: T201
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,217 +0,0 @@
|
||||
"""Settings for migration-mode Django instance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
DEBUG = os.getenv("PAPERLESS_DEBUG", "false").lower() == "true"
|
||||
|
||||
ALLOWED_HOSTS = ["*"]
|
||||
|
||||
# Tap paperless.conf if it's available
|
||||
for path in [
|
||||
os.getenv("PAPERLESS_CONFIGURATION_PATH"),
|
||||
"../paperless.conf",
|
||||
"/etc/paperless.conf",
|
||||
"/usr/local/etc/paperless.conf",
|
||||
]:
|
||||
if path and Path(path).exists():
|
||||
load_dotenv(path)
|
||||
break
|
||||
|
||||
|
||||
def __get_path(
|
||||
key: str,
|
||||
default: str | Path,
|
||||
) -> Path:
|
||||
if key in os.environ:
|
||||
return Path(os.environ[key]).resolve()
|
||||
return Path(default).resolve()
|
||||
|
||||
|
||||
DATA_DIR = __get_path("PAPERLESS_DATA_DIR", BASE_DIR.parent / "data")
|
||||
EXPORT_DIR = __get_path("PAPERLESS_EXPORT_DIR", BASE_DIR.parent / "export")
|
||||
|
||||
|
||||
def _parse_db_settings() -> dict[str, dict[str, Any]]:
|
||||
databases: dict[str, dict[str, Any]] = {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.sqlite3",
|
||||
"NAME": DATA_DIR / "db.sqlite3",
|
||||
"OPTIONS": {},
|
||||
},
|
||||
}
|
||||
if os.getenv("PAPERLESS_DBHOST"):
|
||||
databases["sqlite"] = databases["default"].copy()
|
||||
databases["default"] = {
|
||||
"HOST": os.getenv("PAPERLESS_DBHOST"),
|
||||
"NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
|
||||
"USER": os.getenv("PAPERLESS_DBUSER", "paperless"),
|
||||
"PASSWORD": os.getenv("PAPERLESS_DBPASS", "paperless"),
|
||||
"OPTIONS": {},
|
||||
}
|
||||
if os.getenv("PAPERLESS_DBPORT"):
|
||||
databases["default"]["PORT"] = os.getenv("PAPERLESS_DBPORT")
|
||||
|
||||
if os.getenv("PAPERLESS_DBENGINE") == "mariadb":
|
||||
engine = "django.db.backends.mysql"
|
||||
options = {
|
||||
"read_default_file": "/etc/mysql/my.cnf",
|
||||
"charset": "utf8mb4",
|
||||
"ssl_mode": os.getenv("PAPERLESS_DBSSLMODE", "PREFERRED"),
|
||||
"ssl": {
|
||||
"ca": os.getenv("PAPERLESS_DBSSLROOTCERT"),
|
||||
"cert": os.getenv("PAPERLESS_DBSSLCERT"),
|
||||
"key": os.getenv("PAPERLESS_DBSSLKEY"),
|
||||
},
|
||||
}
|
||||
else:
|
||||
engine = "django.db.backends.postgresql"
|
||||
options = {
|
||||
"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer"),
|
||||
"sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT"),
|
||||
"sslcert": os.getenv("PAPERLESS_DBSSLCERT"),
|
||||
"sslkey": os.getenv("PAPERLESS_DBSSLKEY"),
|
||||
}
|
||||
|
||||
databases["default"]["ENGINE"] = engine
|
||||
databases["default"]["OPTIONS"].update(options)
|
||||
|
||||
if os.getenv("PAPERLESS_DB_TIMEOUT") is not None:
|
||||
timeout = int(os.getenv("PAPERLESS_DB_TIMEOUT"))
|
||||
if databases["default"]["ENGINE"] == "django.db.backends.sqlite3":
|
||||
databases["default"]["OPTIONS"].update({"timeout": timeout})
|
||||
else:
|
||||
databases["default"]["OPTIONS"].update({"connect_timeout": timeout})
|
||||
databases["sqlite"]["OPTIONS"].update({"timeout": timeout})
|
||||
return databases
|
||||
|
||||
|
||||
DATABASES = _parse_db_settings()
|
||||
|
||||
SECRET_KEY = os.getenv(
|
||||
"PAPERLESS_SECRET_KEY",
|
||||
)
|
||||
|
||||
AUTH_PASSWORD_VALIDATORS = [
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
|
||||
},
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
|
||||
},
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
|
||||
},
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
|
||||
},
|
||||
]
|
||||
|
||||
LANGUAGE_CODE = "en-us"
|
||||
TIME_ZONE = "UTC"
|
||||
USE_I18N = True
|
||||
USE_TZ = True
|
||||
CSRF_TRUSTED_ORIGINS: list[str] = []
|
||||
|
||||
INSTALLED_APPS = [
|
||||
"django.contrib.auth",
|
||||
"django.contrib.contenttypes",
|
||||
"django.contrib.sessions",
|
||||
"django.contrib.messages",
|
||||
"django.contrib.staticfiles",
|
||||
"allauth",
|
||||
"allauth.account",
|
||||
"allauth.socialaccount",
|
||||
"allauth.mfa",
|
||||
"paperless_migration",
|
||||
]
|
||||
|
||||
MIDDLEWARE = [
|
||||
"django.middleware.security.SecurityMiddleware",
|
||||
"django.contrib.sessions.middleware.SessionMiddleware",
|
||||
"django.middleware.common.CommonMiddleware",
|
||||
"django.middleware.csrf.CsrfViewMiddleware",
|
||||
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
||||
"django.contrib.messages.middleware.MessageMiddleware",
|
||||
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
||||
"allauth.account.middleware.AccountMiddleware",
|
||||
]
|
||||
|
||||
ROOT_URLCONF = "paperless_migration.urls"
|
||||
|
||||
TEMPLATES = [
|
||||
{
|
||||
"BACKEND": "django.template.backends.django.DjangoTemplates",
|
||||
"DIRS": [
|
||||
BASE_DIR / "paperless_migration" / "templates",
|
||||
BASE_DIR / "documents" / "templates",
|
||||
],
|
||||
"APP_DIRS": True,
|
||||
"OPTIONS": {
|
||||
"context_processors": [
|
||||
"django.template.context_processors.request",
|
||||
"django.contrib.auth.context_processors.auth",
|
||||
"django.contrib.messages.context_processors.messages",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
WSGI_APPLICATION = "paperless_migration.wsgi.application"
|
||||
|
||||
AUTHENTICATION_BACKENDS = [
|
||||
"django.contrib.auth.backends.ModelBackend",
|
||||
"allauth.account.auth_backends.AuthenticationBackend",
|
||||
]
|
||||
|
||||
STATIC_URL = "/static/"
|
||||
STATICFILES_DIRS = [
|
||||
BASE_DIR / ".." / "static",
|
||||
BASE_DIR / "static",
|
||||
BASE_DIR / "documents" / "static",
|
||||
]
|
||||
|
||||
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
||||
|
||||
LOGIN_URL = "/accounts/login/"
|
||||
LOGIN_REDIRECT_URL = "/migration/"
|
||||
LOGOUT_REDIRECT_URL = "/accounts/login/?loggedout=1"
|
||||
|
||||
ACCOUNT_ADAPTER = "allauth.account.adapter.DefaultAccountAdapter"
|
||||
ACCOUNT_AUTHENTICATED_LOGIN_REDIRECTS = False
|
||||
SOCIALACCOUNT_ADAPTER = "allauth.socialaccount.adapter.DefaultSocialAccountAdapter"
|
||||
SOCIALACCOUNT_ENABLED = False
|
||||
|
||||
SESSION_ENGINE = "django.contrib.sessions.backends.db"
|
||||
|
||||
MIGRATION_EXPORT_PATH = __get_path(
|
||||
"PAPERLESS_MIGRATION_EXPORT_PATH",
|
||||
EXPORT_DIR / "manifest.json",
|
||||
)
|
||||
MIGRATION_TRANSFORMED_PATH = __get_path(
|
||||
"PAPERLESS_MIGRATION_TRANSFORMED_PATH",
|
||||
EXPORT_DIR / "manifest.v3.json",
|
||||
)
|
||||
MIGRATION_IMPORTED_PATH = Path(EXPORT_DIR / "import.completed").resolve()
|
||||
|
||||
# One-time access code required for migration logins; stable across autoreload
|
||||
_code = os.getenv("PAPERLESS_MIGRATION_ACCESS_CODE")
|
||||
if not _code:
|
||||
_code = secrets.token_urlsafe(12)
|
||||
os.environ["PAPERLESS_MIGRATION_ACCESS_CODE"] = _code
|
||||
MIGRATION_ACCESS_CODE = _code
|
||||
if os.environ.get("PAPERLESS_MIGRATION_CODE_LOGGED") != "1":
|
||||
logging.getLogger(__name__).warning(
|
||||
"Migration one-time access code: %s",
|
||||
MIGRATION_ACCESS_CODE,
|
||||
)
|
||||
os.environ["PAPERLESS_MIGRATION_CODE_LOGGED"] = "1"
|
||||
@@ -1,77 +0,0 @@
|
||||
{% load i18n static %}
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||
<meta name="author" content="Paperless-ngx project and contributors">
|
||||
<meta name="robots" content="noindex,nofollow">
|
||||
<meta name="color-scheme" content="light">
|
||||
<title>{% translate "Paperless-ngx sign in" %}</title>
|
||||
<link href="{% static 'bootstrap.min.css' %}" rel="stylesheet">
|
||||
<link href="{% static 'base.css' %}" rel="stylesheet">
|
||||
<style>
|
||||
:root, body, .form-control, .form-floating {
|
||||
color-scheme: light;
|
||||
--bs-body-bg: #f5f5f5;
|
||||
--bs-body-color: #212529;
|
||||
--bs-body-color-rgb: 33, 37, 41;
|
||||
--bs-border-color: #dee2e6;
|
||||
--bs-link-color: #17541f;
|
||||
--bs-link-color-rgb: 23, 84, 31;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) { :root { color-scheme: light; } }
|
||||
body {
|
||||
min-height: 100vh;
|
||||
background:
|
||||
radial-gradient(circle at 20% 20%, #eef5ef, #f7fbf7),
|
||||
linear-gradient(120deg, rgba(23, 84, 31, 0.05) 0%, rgba(0,0,0,0) 30%),
|
||||
linear-gradient(300deg, rgba(15, 54, 20, 0.06) 0%, rgba(0,0,0,0) 40%);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body class="d-flex align-items-center justify-content-center text-center p-3">
|
||||
<main class="w-100" style="max-width: 360px;">
|
||||
<form class="form-accounts p-4 rounded-4" id="form-account" method="post">
|
||||
{% csrf_token %}
|
||||
{% include "paperless-ngx/snippets/svg_logo.html" with extra_attrs="width='240' class='logo mb-3'" %}
|
||||
<p class="text-uppercase fw-semibold mb-1 text-secondary small" style="letter-spacing: 0.12rem;">{% translate "Migration Mode" %}</p>
|
||||
|
||||
{% for message in messages %}
|
||||
<div class="alert alert-{{ message.level_tag }} mb-2" role="alert">{{ message }}</div>
|
||||
{% endfor %}
|
||||
|
||||
<p class="mb-3">{% translate "Login with a superuser account to proceed." %}</p>
|
||||
|
||||
{% if form.errors %}
|
||||
<div class="alert alert-danger" role="alert">
|
||||
{% for field, errors in form.errors.items %}
|
||||
{% for error in errors %}
|
||||
{{ error }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% translate "Username" as i18n_username %}
|
||||
{% translate "Password" as i18n_password %}
|
||||
<div class="form-floating form-stacked-top">
|
||||
<input type="text" name="login" id="inputUsername" placeholder="{{ i18n_username }}" class="form-control" autocorrect="off" autocapitalize="none" required autofocus>
|
||||
<label for="inputUsername">{{ i18n_username }}</label>
|
||||
</div>
|
||||
<div class="form-floating form-stacked-middle">
|
||||
<input type="password" name="password" id="inputPassword" placeholder="{{ i18n_password }}" class="form-control" required>
|
||||
<label for="inputPassword">{{ i18n_password }}</label>
|
||||
</div>
|
||||
<div class="form-floating form-stacked-bottom">
|
||||
<input type="text" name="code" id="inputCode" placeholder="One-time code" class="form-control" required>
|
||||
<label for="inputCode">One-time code</label>
|
||||
</div>
|
||||
<p class="mt-2 small fst-italic">{% translate "Code can be found in the startup logs." %}</p>
|
||||
<div class="d-grid mt-3">
|
||||
<button class="btn btn-lg btn-primary" type="submit">{% translate "Sign in" %}</button>
|
||||
</div>
|
||||
</form>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,318 +0,0 @@
|
||||
<!doctype html>
|
||||
{% load static %}
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Paperless-ngx Migration Mode</title>
|
||||
<link rel="stylesheet" href="{% static 'bootstrap.min.css' %}" />
|
||||
<link rel="stylesheet" href="{% static 'base.css' %}" />
|
||||
<style>
|
||||
:root, .form-control {
|
||||
color-scheme: light;
|
||||
--bs-body-bg: #f5f5f5;
|
||||
--bs-body-color: #212529;
|
||||
--bs-body-color-rgb: 33, 37, 41;
|
||||
--bs-border-color: #dee2e6;
|
||||
--bs-link-color: var(--pngx-primary);
|
||||
--bs-link-color-rgb: 23, 84, 31;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) { :root { color-scheme: light; } }
|
||||
|
||||
.btn-primary:disabled {
|
||||
--bs-btn-disabled-bg: #4d7352;
|
||||
--bs-btn-disabled-border-color: #4d7352;
|
||||
}
|
||||
|
||||
body {
|
||||
background:
|
||||
radial-gradient(circle at 20% 20%, #eef5ef, #f7fbf7),
|
||||
linear-gradient(120deg, rgba(23, 84, 31, 0.05) 0%, rgba(0,0,0,0) 30%),
|
||||
linear-gradient(300deg, rgba(15, 54, 20, 0.06) 0%, rgba(0,0,0,0) 40%);
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
svg.logo .text {
|
||||
fill: #161616 !important;
|
||||
}
|
||||
|
||||
.hero-card,
|
||||
.card-step {
|
||||
background: #fff;
|
||||
backdrop-filter: blur(6px);
|
||||
border: 1px solid rgba(23, 84, 31, 0.08);
|
||||
box-shadow: 0 16px 40px rgba(0, 0, 0, 0.06);
|
||||
border-radius: 18px;
|
||||
}
|
||||
|
||||
.status-dot {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.card-step {
|
||||
border-radius: 16px;
|
||||
transition: transform 0.15s ease, box-shadow 0.15s ease;
|
||||
}
|
||||
|
||||
.card-step.done-step {
|
||||
opacity: 0.4;
|
||||
}
|
||||
|
||||
.path-pill {
|
||||
background: rgba(23, 84, 31, 0.08);
|
||||
color: var(--bs-body-color);
|
||||
border-radius: 12px;
|
||||
padding: 0.4rem 0.75rem;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.step-rail {
|
||||
position: relative;
|
||||
height: 4px;
|
||||
background: rgba(23, 84, 31, 0.12);
|
||||
border-radius: 999px;
|
||||
}
|
||||
|
||||
.step-rail .fill {
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: calc({{ export_exists|yesno:'33,0' }}% + {{ transformed_exists|yesno:'33,0' }}% + {{ imported_exists|yesno:'34,0' }}%);
|
||||
max-width: 100%;
|
||||
background: linear-gradient(90deg, #17541f, #2c7a3c);
|
||||
border-radius: 999px;
|
||||
transition: width 0.3s ease;
|
||||
}
|
||||
|
||||
.step-chip {
|
||||
width: 38px;
|
||||
height: 38px;
|
||||
border-radius: 50%;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
font-weight: 700;
|
||||
background: #fff;
|
||||
border: 2px solid rgba(23, 84, 31, 0.25);
|
||||
color: #17541f;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.08);
|
||||
}
|
||||
|
||||
.step-chip.done {
|
||||
background: #17541f;
|
||||
color: #fff;
|
||||
border-color: #17541f;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body class="pb-4">
|
||||
<div class="container py-4">
|
||||
<div class="row justify-content-center mb-4">
|
||||
<div class="col-lg-9">
|
||||
<div class="hero-card p-4">
|
||||
<div class="d-flex flex-wrap align-items-center justify-content-between gap-3">
|
||||
<div class="d-flex align-items-center gap-3">
|
||||
{% include "paperless-ngx/snippets/svg_logo.html" with extra_attrs="width='280' class='logo'" %}
|
||||
<div class="ps-2">
|
||||
<p class="text-uppercase fw-semibold mb-1 text-secondary" style="letter-spacing: 0.12rem;">Migration Mode</p>
|
||||
<h1 class="h3 mb-2 text-primary">Paperless-ngx v2 → v3</h1>
|
||||
<p class="text-muted mb-0">Migrate your data from Paperless-ngx version 2 to version 3.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="text-end">
|
||||
<span class="badge bg-success-subtle text-success border border-success-subtle px-3 py-2">Online</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-4">
|
||||
<div class="d-flex justify-content-between align-items-center mb-2">
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<span class="step-chip {% if export_exists %}done{% endif %}">1</span>
|
||||
<div>
|
||||
<div class="fw-semibold mb-0">Export</div>
|
||||
<small class="text-muted">v2 data</small>
|
||||
</div>
|
||||
</div>
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<span class="step-chip {% if transformed_exists %}done{% endif %}">2</span>
|
||||
<div>
|
||||
<div class="fw-semibold mb-0">Transform</div>
|
||||
<small class="text-muted">to v3 schema</small>
|
||||
</div>
|
||||
</div>
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<span class="step-chip {% if imported_exists %}done{% endif %}">3</span>
|
||||
<div>
|
||||
<div class="fw-semibold mb-0">Import</div>
|
||||
<small class="text-muted">into v3</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="step-rail">
|
||||
<div class="fill"></div>
|
||||
</div>
|
||||
</div>
|
||||
{% if messages %}
|
||||
<div class="mt-4">
|
||||
{% for message in messages %}
|
||||
<div class="alert alert-{{ message.level_tag }} mb-2" role="alert">{{ message }}</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="row g-3 mt-2">
|
||||
<div class="col-md-6">
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<span class="status-dot bg-{{ export_exists|yesno:'success,danger' }}"></span>
|
||||
<div>
|
||||
<div class="fw-semibold">Export file</div>
|
||||
<div class="small text-muted">{{ export_exists|yesno:"Ready,Missing" }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="path-pill mt-2 text-truncate" title="{{ export_path }}">{{ export_path }}</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<span class="status-dot bg-{{ transformed_exists|yesno:'success,warning' }}"></span>
|
||||
<div>
|
||||
<div class="fw-semibold">Transformed file</div>
|
||||
<div class="small text-muted">{{ transformed_exists|yesno:"Ready,Pending" }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="path-pill mt-2 text-truncate" title="{{ transformed_path }}">{{ transformed_path }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row gy-4 justify-content-center">
|
||||
<div class="col-lg-3 col-md-4">
|
||||
<div class="card card-step h-100 {% if export_exists %}done-step{% endif %}">
|
||||
<div class="card-body d-flex flex-column gap-3">
|
||||
<div>
|
||||
<p class="text-uppercase text-muted mb-1 fw-semibold" style="letter-spacing: 0.08rem;">Step 1</p>
|
||||
<h3 class="h5 mb-1">Export (v2)</h3>
|
||||
<p class="small text-muted mb-0">Generate and upload the v2 export file.</p>
|
||||
</div>
|
||||
<div class="mt-auto d-grid gap-2">
|
||||
<form method="post" enctype="multipart/form-data" class="d-flex gap-2 align-items-center">
|
||||
{% csrf_token %}
|
||||
<input class="form-control form-control-sm" type="file" name="export_file" accept=".json" {% if export_exists %}disabled{% endif %} required>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="submit" name="action" value="upload" {% if export_exists %}disabled aria-disabled="true"{% endif %}>Upload</button>
|
||||
</form>
|
||||
<form method="post">
|
||||
{% csrf_token %}
|
||||
<button class="btn btn-primary w-100" type="submit" name="action" value="check" {% if export_exists %}disabled aria-disabled="true"{% endif %}>Re-check export</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-3 col-md-4">
|
||||
<div class="card card-step h-100 {% if transformed_exists %}done-step{% endif %}">
|
||||
<div class="card-body d-flex flex-column gap-3">
|
||||
<div>
|
||||
<p class="text-uppercase text-muted mb-1 fw-semibold" style="letter-spacing: 0.08rem;">Step 2</p>
|
||||
<h3 class="h5 mb-1">Transform</h3>
|
||||
<p class="small text-muted mb-0">Convert the export into the v3-ready structure.</p>
|
||||
</div>
|
||||
<div class="mt-auto">
|
||||
<form method="post">
|
||||
{% csrf_token %}
|
||||
<button
|
||||
class="btn btn-outline-primary w-100"
|
||||
type="submit"
|
||||
name="action"
|
||||
value="transform"
|
||||
{% if not export_exists or transformed_exists %}disabled aria-disabled="true"{% endif %}
|
||||
>
|
||||
Transform export
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-3 col-md-4">
|
||||
<div class="card card-step h-100 {% if imported_exists %}done-step{% endif %}">
|
||||
<div class="card-body d-flex flex-column gap-3">
|
||||
<div>
|
||||
<p class="text-uppercase text-muted mb-1 fw-semibold" style="letter-spacing: 0.08rem;">Step 3</p>
|
||||
<h3 class="h5 mb-1">Import (v3)</h3>
|
||||
<p class="small text-muted mb-0">Load the transformed data into your v3 instance.</p>
|
||||
</div>
|
||||
<div class="mt-auto">
|
||||
<form method="post">
|
||||
{% csrf_token %}
|
||||
<button
|
||||
class="btn btn-outline-secondary w-100"
|
||||
type="submit"
|
||||
name="action"
|
||||
value="import"
|
||||
{% if not transformed_exists or imported_exists %}disabled aria-disabled="true"{% endif %}
|
||||
>
|
||||
Import transformed data
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row justify-content-center mt-4">
|
||||
<div class="col-lg-9">
|
||||
{% if not export_exists %}
|
||||
<div class="alert alert-info mb-3">
|
||||
<div class="fw-semibold mb-1">Export file not found</div>
|
||||
<div class="small">
|
||||
Run the v2 export from your Paperless instance, e.g.:
|
||||
<code>docker run --rm ghcr.io/paperless-ngx/paperless-ngx:2.20.6 document_exporter --data-only</code>
|
||||
(see <a href="https://docs.paperless-ngx.com/administration/#exporter" target="_blank" rel="noopener noreferrer">documentation</a>). Once the <code>manifest.json</code> is in-place, upload it or (especially for larger files) place it directly at the expected location and click “Re-check export”.
|
||||
<p class="mt-2 mb-0 text-danger fst-italic">⚠️ The export must be generated with version Paperless-ngx v2.20.6</p>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="card card-step">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between align-items-center mb-2">
|
||||
<div class="fw-semibold">Migration console</div>
|
||||
<span class="badge bg-secondary-subtle text-secondary border border-secondary-subtle">Live output</span>
|
||||
</div>
|
||||
<pre id="migration-log" class="mb-0" style="background:#0f1a12;color:#d1e7d6;border-radius:12px;min-height:180px;padding:12px;font-size:0.9rem;overflow:auto;">Ready</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% if stream_action %}
|
||||
<script>
|
||||
(() => {
|
||||
const logEl = document.getElementById('migration-log');
|
||||
if (!logEl) return;
|
||||
const streamUrl = "{% if stream_action == 'import' %}{% url 'import_stream' %}{% else %}{% url 'transform_stream' %}{% endif %}";
|
||||
const donePrefix = "{{ stream_action|capfirst }} finished";
|
||||
const evt = new EventSource(streamUrl);
|
||||
const append = (line) => {
|
||||
logEl.textContent += `\n${line}`;
|
||||
logEl.scrollTop = logEl.scrollHeight;
|
||||
};
|
||||
evt.onmessage = (e) => {
|
||||
append(e.data);
|
||||
if (e.data.startsWith(donePrefix)) {
|
||||
setTimeout(() => window.location.reload(), 500);
|
||||
}
|
||||
};
|
||||
evt.onerror = () => {
|
||||
append('[connection closed]');
|
||||
evt.close();
|
||||
};
|
||||
})();
|
||||
</script>
|
||||
{% endif %}
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,21 +0,0 @@
|
||||
from django.conf import settings
|
||||
from django.conf.urls.static import static
|
||||
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
|
||||
from django.urls import include
|
||||
from django.urls import path
|
||||
|
||||
from paperless_migration import views
|
||||
|
||||
urlpatterns = [
|
||||
path("accounts/login/", views.migration_login, name="account_login"),
|
||||
path("accounts/", include("allauth.urls")),
|
||||
path("migration/", views.migration_home, name="migration_home"),
|
||||
path("migration/transform/stream", views.transform_stream, name="transform_stream"),
|
||||
path("migration/import/stream", views.import_stream, name="import_stream"),
|
||||
# redirect root to migration home
|
||||
path("", views.migration_home, name="migration_home"),
|
||||
]
|
||||
|
||||
if settings.DEBUG:
|
||||
urlpatterns += staticfiles_urlpatterns()
|
||||
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
|
||||
@@ -1,269 +0,0 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth import authenticate
|
||||
from django.contrib.auth import login
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.http import HttpResponseForbidden
|
||||
from django.http import StreamingHttpResponse
|
||||
from django.shortcuts import redirect
|
||||
from django.shortcuts import render
|
||||
from django.views.decorators.http import require_http_methods
|
||||
|
||||
from paperless_migration import settings
|
||||
|
||||
|
||||
@login_required
|
||||
@require_http_methods(["GET", "POST"])
|
||||
def migration_home(request):
|
||||
if not request.session.get("migration_code_ok"):
|
||||
return HttpResponseForbidden("Access code required")
|
||||
if not request.user.is_superuser:
|
||||
return HttpResponseForbidden("Superuser access required")
|
||||
|
||||
export_path = Path(settings.MIGRATION_EXPORT_PATH)
|
||||
transformed_path = Path(settings.MIGRATION_TRANSFORMED_PATH)
|
||||
imported_marker = Path(settings.MIGRATION_IMPORTED_PATH)
|
||||
|
||||
if request.method == "POST":
|
||||
action = request.POST.get("action")
|
||||
if action == "check":
|
||||
messages.success(request, "Checked export paths.")
|
||||
elif action == "transform":
|
||||
messages.info(request, "Starting transform… live output below.")
|
||||
request.session["start_stream_action"] = "transform"
|
||||
if imported_marker.exists():
|
||||
imported_marker.unlink()
|
||||
elif action == "upload":
|
||||
upload = request.FILES.get("export_file")
|
||||
if not upload:
|
||||
messages.error(request, "No file selected.")
|
||||
else:
|
||||
try:
|
||||
export_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with export_path.open("wb") as dest:
|
||||
for chunk in upload.chunks():
|
||||
dest.write(chunk)
|
||||
messages.success(request, f"Uploaded to {export_path}.")
|
||||
except Exception as exc:
|
||||
messages.error(request, f"Failed to save file: {exc}")
|
||||
elif action == "import":
|
||||
messages.info(request, "Starting import… live output below.")
|
||||
request.session["start_stream_action"] = "import"
|
||||
else:
|
||||
messages.error(request, "Unknown action.")
|
||||
return redirect("migration_home")
|
||||
|
||||
stream_action = request.session.pop("start_stream_action", None)
|
||||
context = {
|
||||
"export_path": export_path,
|
||||
"export_exists": export_path.exists(),
|
||||
"transformed_path": transformed_path,
|
||||
"transformed_exists": transformed_path.exists(),
|
||||
"imported_exists": imported_marker.exists(),
|
||||
"stream_action": stream_action,
|
||||
}
|
||||
return render(request, "paperless_migration/migration_home.html", context)
|
||||
|
||||
|
||||
@require_http_methods(["GET", "POST"])
|
||||
def migration_login(request):
|
||||
if request.method == "POST":
|
||||
username = request.POST.get("login", "")
|
||||
password = request.POST.get("password", "")
|
||||
code = request.POST.get("code", "")
|
||||
|
||||
if not code or code != settings.MIGRATION_ACCESS_CODE:
|
||||
messages.error(request, "One-time code is required.")
|
||||
return redirect("account_login")
|
||||
|
||||
user = authenticate(request, username=username, password=password)
|
||||
if user is None:
|
||||
messages.error(request, "Invalid username or password.")
|
||||
return redirect("account_login")
|
||||
|
||||
if not user.is_superuser:
|
||||
messages.error(request, "Superuser access required.")
|
||||
return redirect("account_login")
|
||||
|
||||
login(request, user)
|
||||
request.session["migration_code_ok"] = True
|
||||
return redirect(settings.LOGIN_REDIRECT_URL)
|
||||
|
||||
return render(request, "account/login.html")
|
||||
|
||||
|
||||
@login_required
|
||||
@require_http_methods(["GET"])
|
||||
def transform_stream(request):
|
||||
if not request.session.get("migration_code_ok"):
|
||||
return HttpResponseForbidden("Access code required")
|
||||
if not request.user.is_superuser:
|
||||
return HttpResponseForbidden("Superuser access required")
|
||||
|
||||
input_path = Path(settings.MIGRATION_EXPORT_PATH)
|
||||
output_path = Path(settings.MIGRATION_TRANSFORMED_PATH)
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"paperless_migration.scripts.transform",
|
||||
"--input",
|
||||
str(input_path),
|
||||
"--output",
|
||||
str(output_path),
|
||||
]
|
||||
|
||||
def event_stream():
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
bufsize=1,
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
yield "data: Starting transform...\n\n"
|
||||
if process.stdout:
|
||||
for line in process.stdout:
|
||||
yield f"data: {line.rstrip()}\n\n"
|
||||
process.wait()
|
||||
yield f"data: Transform finished with code {process.returncode}\n\n"
|
||||
finally:
|
||||
if process and process.poll() is None:
|
||||
process.kill()
|
||||
|
||||
return StreamingHttpResponse(
|
||||
event_stream(),
|
||||
content_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@login_required
|
||||
@require_http_methods(["GET"])
|
||||
def import_stream(request):
|
||||
if not request.session.get("migration_code_ok"):
|
||||
return HttpResponseForbidden("Access code required")
|
||||
if not request.user.is_superuser:
|
||||
return HttpResponseForbidden("Superuser access required")
|
||||
|
||||
export_path = Path(settings.MIGRATION_EXPORT_PATH)
|
||||
transformed_path = Path(settings.MIGRATION_TRANSFORMED_PATH)
|
||||
imported_marker = Path(settings.MIGRATION_IMPORTED_PATH)
|
||||
manage_path = Path(settings.BASE_DIR) / "manage.py"
|
||||
source_dir = export_path.parent
|
||||
|
||||
env = os.environ.copy()
|
||||
env["DJANGO_SETTINGS_MODULE"] = "paperless.settings"
|
||||
env["PAPERLESS_MIGRATION_MODE"] = "0"
|
||||
|
||||
def event_stream():
|
||||
if not export_path.exists():
|
||||
yield "data: Missing export manifest.json; upload or re-check export.\n\n"
|
||||
return
|
||||
if not transformed_path.exists():
|
||||
yield "data: Missing transformed manifest.v3.json; run transform first.\n\n"
|
||||
return
|
||||
|
||||
backup_path: Path | None = None
|
||||
try:
|
||||
backup_fd, backup_name = tempfile.mkstemp(
|
||||
prefix="manifest.v2.",
|
||||
suffix=".json",
|
||||
dir=source_dir,
|
||||
)
|
||||
os.close(backup_fd)
|
||||
backup_path = Path(backup_name)
|
||||
shutil.copy2(export_path, backup_path)
|
||||
shutil.copy2(transformed_path, export_path)
|
||||
except Exception as exc:
|
||||
yield f"data: Failed to prepare import manifest: {exc}\n\n"
|
||||
return
|
||||
|
||||
def run_cmd(args, label):
|
||||
yield f"data: {label}\n\n"
|
||||
process = subprocess.Popen(
|
||||
args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
bufsize=1,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
try:
|
||||
if process.stdout:
|
||||
for line in process.stdout:
|
||||
yield f"data: {line.rstrip()}\n\n"
|
||||
process.wait()
|
||||
return process.returncode
|
||||
finally:
|
||||
if process and process.poll() is None:
|
||||
process.kill()
|
||||
|
||||
wipe_cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"paperless_migration.scripts.wipe_db",
|
||||
]
|
||||
migrate_cmd = [
|
||||
sys.executable,
|
||||
str(manage_path),
|
||||
"migrate",
|
||||
"--noinput",
|
||||
]
|
||||
import_cmd = [
|
||||
sys.executable,
|
||||
str(manage_path),
|
||||
"document_importer",
|
||||
str(source_dir),
|
||||
"--data-only",
|
||||
]
|
||||
try:
|
||||
wipe_code = yield from run_cmd(
|
||||
wipe_cmd,
|
||||
"Wiping database...",
|
||||
)
|
||||
if wipe_code != 0:
|
||||
yield f"data: Wipe finished with code {wipe_code}\n\n"
|
||||
return
|
||||
|
||||
migrate_code = yield from run_cmd(
|
||||
migrate_cmd,
|
||||
"Running migrations...",
|
||||
)
|
||||
if migrate_code != 0:
|
||||
yield f"data: Migrate finished with code {migrate_code}\n\n"
|
||||
return
|
||||
|
||||
import_code = yield from run_cmd(
|
||||
import_cmd,
|
||||
"Starting import...",
|
||||
)
|
||||
if import_code == 0:
|
||||
imported_marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
imported_marker.write_text("ok\n", encoding="utf-8")
|
||||
yield f"data: Import finished with code {import_code}\n\n"
|
||||
finally:
|
||||
if backup_path and backup_path.exists():
|
||||
try:
|
||||
shutil.move(backup_path, export_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return StreamingHttpResponse(
|
||||
event_stream(),
|
||||
content_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
@@ -1,7 +0,0 @@
|
||||
import os
|
||||
|
||||
from django.core.wsgi import get_wsgi_application
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless_migration.settings")
|
||||
|
||||
application = get_wsgi_application()
|
||||
Reference in New Issue
Block a user