From 6c138a21d4392539c379eac6e0c9e4976206c696 Mon Sep 17 00:00:00 2001 From: Orce MARINKOVSKI Date: Thu, 26 Dec 2024 17:10:02 +0100 Subject: [PATCH] Development: devcontainer improvements (#8553) --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- .devcontainer/devcontainer.json | 28 +- ...ocker-compose.devcontainer.sqlite-tika.yml | 13 +- .devcontainer/vscode/launch.json | 69 ++-- .devcontainer/vscode/tasks.json | 319 ++++++++++++------ .gitignore | 6 + docs/development.md | 20 ++ 6 files changed, 299 insertions(+), 156 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 337c96e3b..34446caea 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,14 +3,26 @@ "dockerComposeFile": "docker-compose.devcontainer.sqlite-tika.yml", "service": "paperless-development", "workspaceFolder": "/usr/src/paperless/paperless-ngx", - "postCreateCommand": "pipenv install --dev && pipenv run pre-commit install && pipenv shell", + "postCreateCommand": "pipenv install --dev && pipenv run pre-commit install", "customizations": { "vscode": { - "extensions": [ - "mhutchie.git-graph", - "ms-python.python" - ] + "extensions": [ + "mhutchie.git-graph", + "ms-python.python", + "ms-vscode.js-debug-nightly", + "eamodio.gitlens", + "yzhang.markdown-all-in-one" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/src/paperless/paperless-ngx/.venv/bin/python", + "python.pythonPath": "/usr/src/paperless/paperless-ngx/.venv/bin/python", + "python.terminal.activateEnvInCurrentTerminal": true, + "editor.formatOnPaste": false, + "editor.formatOnSave": true, + "editor.formatOnType": true, + "files.trimTrailingWhitespace": true + } } - }, - "remoteUser": "paperless" - } + }, + "remoteUser": "paperless" + } diff --git a/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml b/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml index cd978ebcb..7209339e1 100644 --- a/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml +++ b/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml @@ -27,7 +27,7 @@ services: image: docker.io/library/redis:7 restart: unless-stopped volumes: - - redisdata:/data + - ./redisdata:/data # No ports need to be exposed; the VSCode DevContainer plugin manages them. paperless-development: @@ -43,14 +43,16 @@ services: volumes: - ..:/usr/src/paperless/paperless-ngx:delegated - ../.devcontainer/vscode:/usr/src/paperless/paperless-ngx/.vscode:delegated # VSCode config files - - pipenv:/usr/src/paperless/paperless-ngx/.venv # Pipenv environment persisted in volume + - pipenv:/usr/src/paperless/paperless-ngx/.venv - /usr/src/paperless/paperless-ngx/src/documents/static/frontend # Static frontend files exist only in container - /usr/src/paperless/paperless-ngx/src/.pytest_cache - /usr/src/paperless/paperless-ngx/.ruff_cache - /usr/src/paperless/paperless-ngx/htmlcov - /usr/src/paperless/paperless-ngx/.coverage - - data:/usr/src/paperless/paperless-ngx/data - - media:/usr/src/paperless/paperless-ngx/media + - ./data:/usr/src/paperless/paperless-ngx/data + - ./media:/usr/src/paperless/paperless-ngx/media + - ./consume:/usr/src/paperless/paperless-ngx/consume + - ~/.gitconfig:/usr/src/paperless/.gitconfig:ro environment: PAPERLESS_REDIS: redis://broker:6379 PAPERLESS_TIKA_ENABLED: 1 @@ -78,7 +80,4 @@ services: restart: unless-stopped volumes: - data: - media: - redisdata: pipenv: diff --git a/.devcontainer/vscode/launch.json b/.devcontainer/vscode/launch.json index 2c50af44b..b21226630 100644 --- a/.devcontainer/vscode/launch.json +++ b/.devcontainer/vscode/launch.json @@ -2,42 +2,53 @@ "version": "0.2.0", "configurations": [ { - "name": "manage.py runserver", + "name": "Chrome: Debug Angular Frontend", + "type": "chrome", + "request": "launch", + "url": "http://localhost:4200", + "webRoot": "${workspaceFolder}/src-ui", + "preLaunchTask": "Start: Frontend Angular" + }, + { + "name": "Debug: Backend Server (manage.py runserver)", "type": "python", "request": "launch", "program": "${workspaceFolder}/src/manage.py", - "console": "integratedTerminal", - "justMyCode": true, - "args": ["runserver"], - "django": true - }, - { - "name": "manage.py document_consumer", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/src/manage.py", - "console": "integratedTerminal", - "justMyCode": true, - "args": ["document_consumer"], - "django": true - }, - { - "name": "celery", - "type": "python", - "cwd": "${workspaceFolder}/src", - "request": "launch", - "module": "celery", + "args": [ + "runserver" + ], + "django": true, "console": "integratedTerminal", "env": { "PYTHONPATH": "${workspaceFolder}/src" - }, + }, + "python": "${workspaceFolder}/.venv/bin/python" + }, + { + "name": "Debug: Consumer Service (manage.py document_consumer)", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/src/manage.py", "args": [ - "-A", - "paperless", - "worker", - "-l", - "DEBUG" - ] + "document_consumer" + ], + "django": true, + "console": "integratedTerminal", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + }, + "python": "${workspaceFolder}/.venv/bin/python" + } + ], + "compounds": [ + { + "name": "Debug: FullStack", + "configurations": [ + "Chrome: Debug Angular Frontend", + "Debug: Backend Server (manage.py runserver)", + "Debug: Consumer Service (manage.py document_consumer)" + ], + "preLaunchTask": "Start: Celery Worker" } ] } diff --git a/.devcontainer/vscode/tasks.json b/.devcontainer/vscode/tasks.json index fa27ba6bc..d7eaf9d3f 100644 --- a/.devcontainer/vscode/tasks.json +++ b/.devcontainer/vscode/tasks.json @@ -1,27 +1,82 @@ { "version": "2.0.0", "tasks": [ - { - "label": "manage.py document_consumer", - "type": "shell", - "command": "pipenv run python manage.py document_consumer", - "group": "build", - "presentation": { - "echo": true, - "reveal": "always", - "focus": false, - "panel": "shared", - "showReuseMessage": false, - "clear": true, - "revealProblems": "onProblem" - }, - "options": { - "cwd": "${workspaceFolder}/src" - } - + { + "label": "Start: Celery Worker", + "type": "shell", + "command": "pipenv run celery --app paperless worker -l DEBUG", + "isBackground": true, + "options": { + "cwd": "${workspaceFolder}/src" + }, + "problemMatcher": [ + { + "owner": "custom", + "pattern": [ + { + "regexp": ".", + "file": 1, + "location": 2, + "message": 3 + } + ], + "background": { + "activeOnStart": true, + "beginsPattern": "celery.*", + "endsPattern": "ready" + } + } + ], + "detail": "" }, { - "label": "manage.py runserver", + "label": "Start: Frontend Angular", + "type": "shell", + "command": "npm start", + "isBackground": true, + "options": { + "cwd": "${workspaceFolder}/src-ui" + }, + "problemMatcher": [ + { + "owner": "custom", + "pattern": [ + { + "regexp": ".", + "file": 1, + "location": 2, + "message": 3 + } + ], + "background": { + "activeOnStart": true, + "beginsPattern": ".*", + "endsPattern": "Compiled successfully" + } + } + ], + "detail": "triggered also by launch option Chrome: Debug Angular Frontend" + }, + { + "label": "Start: Consumer Service (manage.py document_consumer)", + "type": "shell", + "command": "pipenv run python manage.py document_consumer", + "group": "build", + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + }, + { + "label": "Start: Backend Server (manage.py runserver)", "type": "shell", "command": "pipenv run python manage.py runserver", "group": "build", @@ -37,100 +92,140 @@ "options": { "cwd": "${workspaceFolder}/src" } - + }, + { + "label": "Maintenance: manage.py migrate", + "type": "shell", + "command": "pipenv run python manage.py migrate", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" }, - { - "label": "Maintenance: manage.py migrate", - "type": "shell", - "command": "pipenv run python manage.py migrate", - "group": "none", - "presentation": { - "echo": true, - "reveal": "always", - "focus": true, - "panel": "shared", - "showReuseMessage": false, - "clear": true, - "revealProblems": "onProblem" + "options": { + "cwd": "${workspaceFolder}/src" + } }, - "options": { - "cwd": "${workspaceFolder}/src" + { + "label": "Maintenance: Build Documentation", + "type": "shell", + "command": "pipenv run mkdocs build --config-file mkdocs.yml && pipenv run mkdocs serve", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}" + } + }, + { + "label": "Maintenance: manage.py createsuperuser", + "type": "shell", + "command": "pipenv run python manage.py createsuperuser", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + }, + { + "label": "Maintenance: Install Angular CLI", + "type": "shell", + "command": "npm ci && ./node_modules/.bin/ng build --configuration production", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src-ui" + } + }, + { + "label": "Maintenance: Compile frontend for production", + "type": "shell", + "command": "npm ci && ./node_modules/.bin/ng build --configuration production", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src-ui" + } + }, + { + "label": "Maintenance: recreate .venv", + "type": "shell", + "command": "rm -R -v .venv/* || pipenv install --dev", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}" + } + }, + { + "label": "Maintenance: Install Frontend Dependencies", + "type": "npm", + "script": "install", + "path": "src-ui", + "group": "clean", + "problemMatcher": [], + "detail": "install dependencies from package" + }, + { + "label": "Project Setup: Run all Init Tasks", + "dependsOrder": "sequence", + "dependsOn": [ + "Maintenance: manage.py migrate", + "Maintenance: manage.py createsuperuser", + "Maintenance: Compile frontend for production" + ] + }, + { + "label": "Project Start: Run all Services", + "dependsOn": [ + "Start: Celery Worker", + "Start: Consumer Service (manage.py document_consumer)", + "Start: Backend Server (manage.py runserver)" + ] } - }, - { - "label": "Maintenance: manage.py createsuperuser", - "type": "shell", - "command": "pipenv run python manage.py createsuperuser", - "group": "none", - "presentation": { - "echo": true, - "reveal": "always", - "focus": true, - "panel": "shared", - "showReuseMessage": false, - "clear": true, - "revealProblems": "onProblem" - }, - "options": { - "cwd": "${workspaceFolder}/src" - } - }, - { - "label": "compile frontend", - "type": "shell", - "command": "npm ci && ./node_modules/.bin/ng build --configuration production", - "group": "none", - "presentation": { - "echo": true, - "reveal": "always", - "focus": true, - "panel": "shared", - "showReuseMessage": false, - "clear": true, - "revealProblems": "onProblem" - }, - "options": { - "cwd": "${workspaceFolder}/src-ui" - } - }, - { - "label": "Maintenance: recreate .venv", - "type": "shell", - "command": "rm -R -v .venv/* || pipenv install --dev", - "group": "none", - "presentation": { - "echo": true, - "reveal": "always", - "focus": true, - "panel": "shared", - "showReuseMessage": false, - "clear": true, - "revealProblems": "onProblem" - }, - "options": { - "cwd": "${workspaceFolder}" - } - }, - { - "label": "Celery Worker", - "type": "shell", - "command": "pipenv run celery --app paperless worker -l DEBUG", - "group": { - "kind": "build", - "isDefault": true - }, - "presentation": { - "echo": true, - "reveal": "always", - "focus": true, - "panel": "shared", - "showReuseMessage": false, - "clear": true, - "revealProblems": "onProblem" - }, - "options": { - "cwd": "${workspaceFolder}/src" - } - } ] - } +} diff --git a/.gitignore b/.gitignore index 3351a924b..0fd9d7bcd 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,9 @@ scripts/nuke # celery schedule file celerybeat-schedule* + +# ignore .devcontainer sub folders +/.devcontainer/consume/ +/.devcontainer/data/ +/.devcontainer/media/ +/.devcontainer/redisdata/ diff --git a/docs/development.md b/docs/development.md index 44a2f6986..b49a42a2f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -450,3 +450,23 @@ def myparser_consumer_declaration(sender, **kwargs): mime types have many extensions associated with them and the Python methods responsible for guessing the extension do not always return the same value. + +## Using Visual Studio Code devcontainer + +Another easy way to get started with development is to use Visual Studio +Code devcontainers. This approach will create a preconfigured development +environment with all of the required tools and dependencies. +[Learn more about devcontainers](https://code.visualstudio.com/docs/devcontainers/containers). + +To get started: + +1. Clone the repository on your machine and open the Paperless-ngx folder in VS Code. + +2. VS Code will prompt you with "Reopen in container". Do so and wait for the environment to start. + +3. Initialize the project by running the task **Project Setup: Run all Init Tasks**. This + will initialize the database tables and create a superuser. Then you can compile the front end + for production or run the frontend in debug mode. + +4. The project is ready for debugging, start either run the fullstack debug or individual debug + processes. Yo spin up the project without debugging run the task **Project Start: Run all Services**