From 43ad2e59b18a5c5d6cf1dc5129edb8ffc119c412 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 22 Dec 2020 18:46:47 +0100 Subject: [PATCH 01/20] add basic ansible role for debian deployment Currently only Debian 10 buster is supported. Other Debian versions, Ubuntu and derivates should be easy to integrate. Database deployment is considered out-of-scope and deferred to the user. Provides basic upgrade support between releases. --- ansible/defaults/main.yml | 39 +++++ ansible/tasks/main.yml | 350 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 389 insertions(+) create mode 100644 ansible/defaults/main.yml create mode 100644 ansible/tasks/main.yml diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml new file mode 100644 index 000000000..0fa03b47a --- /dev/null +++ b/ansible/defaults/main.yml @@ -0,0 +1,39 @@ +--- +paperlessng_version: 0.9.8 +paperlessng_directory: /opt/paperless-ng +paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" +paperlessng_data_dir: "{{ paperlessng_directory }}/data" +paperlessng_media_root: "{{ paperlessng_directory }}/media" +paperlessng_static_dir: "{{ paperlessng_directory }}/static" +paperlessng_filename_format: +paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv" + +paperlessng_ocr_languages: + - eng +paperlessng_time_zone: Europe/Berlin +paperlessng_ocrmypdf_args: --optimize 1 +# TODO Does optimze==1 really work with jbig2enc? +# https://ocrmypdf.readthedocs.io/en/latest/jbig2.html#lossy-mode-jbig2 +# Documentation states -O1 only applies lossless transformations +# https://ocrmypdf.readthedocs.io/en/latest/optimizer.html#lossless-optimizations +paperlessng_use_jbig2enc: true + +paperlessng_superuser_name: paperlessng +paperlessng_superuser_email: paperlessng@example.com +paperlessng_superuser_password: paperlessng + +paperlessng_system_user: paperlessng +paperlessng_system_group: paperlessng + +paperlessng_listen_address: 127.0.0.1 +paperlessng_listen_port: 8000 + +paperlessng_redis_host: localhost +paperlessng_redis_port: 6379 + +paperlessng_db_type: sqlite # or postgresql +paperlessng_db_host: localhost +paperlessng_db_port: 5432 +paperlessng_db_name: paperlessng +paperlessng_db_user: paperlessng +paperlessng_db_pass: paperlessng \ No newline at end of file diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml new file mode 100644 index 000000000..b427482b2 --- /dev/null +++ b/ansible/tasks/main.yml @@ -0,0 +1,350 @@ +--- +- name: verify operating system + fail: + msg: Sorry, only Debian 10 supported at the moment. + when: not(ansible_distribution == 'Debian' and ansible_distribution_version == '10') + +- name: install base dependencies + apt: + update_cache: yes + pkg: + # paperless-ng + - python3-dev + - python3-pip + - imagemagick + - unpaper + - ghostscript + - optipng + - tesseract-ocr + - gnupg + - libpoppler-cpp-dev + - libmagic-dev + - libpq-dev + # OCRmyPDF + - icc-profiles-free + - qpdf + - liblept5 + - libxml2 + - pngquant + - zlib1g + # dev + - build-essential + - python3-setuptools + - python3-wheel + - python3-virtualenv + +- name: install ocr languages + apt: + pkg: "{{ paperlessng_ocr_languages | map('regex_replace', '^(.*)$', 'tesseract-ocr-\\1') | list }}" + +- name: set up notesalexp repository key (for jbig2enc) + apt_key: + url: https://notesalexp.org/debian/alexp_key.asc + state: present + when: paperlessng_use_jbig2enc + +- name: set up notesalexp repository (for jbig2enc) + apt_repository: + repo: deb https://notesalexp.org/debian/buster/ buster main + state: present + when: paperlessng_use_jbig2enc + +- name: set up notesalexp repository pinning + copy: + content: | + Package: * + Pin: release o=notesalexp.org + Pin-Priority: 1 + + Package: jbig2enc + Pin: release o=notesalexp.org + Pin-Priority: 500 + dest: /etc/apt/preferences.d/notesalexp + when: paperlessng_use_jbig2enc + +- name: install jbig2enc + apt: + pkg: jbig2enc + update_cache: yes + when: paperlessng_use_jbig2enc + +- name: install redis + apt: + pkg: redis-server + when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' + +- name: enable redis + systemd: + name: redis-server + enabled: yes + masked: no + state: started + when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' + +- name: check for paperless-ng installation + command: + cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' + changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + failed_when: false + ignore_errors: yes + register: paperlessng_current_version + +- name: backup current paperless-ng installation + copy: + src: "{{ paperlessng_directory }}" + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" + remote_src: yes + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: download paperless-ng + get_url: + url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + dest: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: create paperless-ng directories + file: + path: "{{ item }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: 0750 + recurse: yes + with_items: + - "{{ paperlessng_directory }}" + - "{{ paperlessng_consumption_dir }}" + - "{{ paperlessng_data_dir }}" + - "{{ paperlessng_media_root }}" + - "{{ paperlessng_static_dir }}" + +- name: create temporary directory + tempfile: + state: directory + register: tempdir + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: extract paperless-ng + unarchive: + src: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz + dest: "{{ tempdir.path }}" + remote_src: yes + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: move paperless-ng + command: + cmd: "cp -R {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" + args: + warn: false + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: remove temporary directory + file: + path: "{{ tempdir.path }}" + state: absent + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: configure paperless-ng + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - regexp: "^#?PAPERLESS_REDIS=" + line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}" + - regexp: "^#?PAPERLESS_CONSUMPTION_DIR=" + line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}" + - regexp: "^#?PAPERLESS_DATA_DIR=" + line: "PAPERLESS_DATA_DIR={{ paperlessng_data_dir }}" + - regexp: "^#?PAPERLESS_MEDIA_ROOT=" + line: "PAPERLESS_MEDIA_ROOT={{ paperlessng_media_root }}" + - regexp: "^#?PAPERLESS_STATICDIR=" + line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}" + - regexp: "^#?PAPERLESS_FILENAME_FORMAT=" + line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" + - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" + line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" + - regexp: "^#PAPERLESS_OCR_USER_ARG=" + # TODO JSON dict required in conf? + # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings + line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + - regexp: "^#?PAPERLESS_TIME_ZONE=" + line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" + no_log: true + +- name: configure paperless-ng database [sqlite] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "^#?PAPERLESS_DBHOST=" + state: absent + when: paperlessng_db_type == 'sqlite' + +- name: configure paperless-ng database [postgresql] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - regexp: "^#?PAPERLESS_DBHOST=" + line: "PAPERLESS_DBHOST={{ paperlessng_db_host }}" + - regexp: "^#?PAPERLESS_DBPORT=" + line: "PAPERLESS_DBPORT={{ paperlessng_db_port }}" + - regexp: "^#?PAPERLESS_DBNAME=" + line: "PAPERLESS_DBNAME={{ paperlessng_db_name }}" + - regexp: "^#?PAPERLESS_DBUSER=" + line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}" + - regexp: "^#?PAPERLESS_DBPASS=" + line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" + when: paperlessng_db_type == 'postgresql' + no_log: true + +- name: create paperlessng venv + command: + cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3" + creates: "{{ paperlessng_virtualenv }}" + +- name: install paperlessng requirements + pip: + requirements: "{{ paperlessng_directory }}/requirements.txt" + virtualenv: "{{ paperlessng_virtualenv }}" + extra_args: --upgrade + +- name: collect static files + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --clear --no-input" + +- name: create database schema + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + register: database_schema + changed_when: '"No migrations to apply." not in database_schema.stdout' + +- name: create first paperless user + # "manage.py createsuperuser" only works on interactive TTYs + command: | + {{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c " + from django.contrib.auth.models import User + from django.contrib.auth.hashers import get_hasher + + if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): + user = User.objects.get(username='{{ paperlessng_superuser_name }}') + old = user.__dict__.copy() + + user.is_superuser = True + user.email = '{{ paperlessng_superuser_email }}' + user.set_password('{{ paperlessng_superuser_password }}') + user.save() + new = user.__dict__ + + algorithm, iterations, old_salt, old_hash = old['password'].split('$') + new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) + _, _, _, new_hash = new_password_old_salt.split('$') + if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): + print('changed') + else: + User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') + print('changed') + " + register: superuser + changed_when: superuser.stdout == 'changed' + no_log: true + +- name: configure ghostscript for PDF + lineinfile: + path: "/etc/ImageMagick-6/policy.xml" + regexp: '' + line: '' + +- name: create paperless group + group: + name: "{{ paperlessng_system_group }}" + +- name: create paperless user + user: + name: "{{ paperlessng_system_user }}" + groups: + - "{{ paperlessng_system_group }}" + shell: /usr/sbin/nologin + # GNUPG_HOME required due to paperless db.py + create_home: yes + +- name: configure systemd services + ini_file: + path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" + section: "{{ item[1].section }}" + option: "{{ item[1].option }}" + value: "{{ item[1].value }}" + with_nested: + - [ + paperless-consumer.service, + paperless-scheduler.service, + paperless-webserver.service, + ] + - [ + { + section: "Service", + option: "User", + value: "{{ paperlessng_system_user }}", + }, + { + section: "Service", + option: "Group", + value: "{{ paperlessng_system_group }}", + }, + { + section: "Service", + option: "WorkingDirectory", + value: "{{ paperlessng_directory }}/src", + }, + ] + +- name: configure paperless-consumer service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-consumer.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py document_consumer" + +- name: configure paperless-scheduler service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-scheduler.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py qcluster" + +- name: configure paperless-webserver service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}" + +- name: copy systemd services + copy: + src: "{{ paperlessng_directory }}/scripts/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + remote_src: yes + with_items: + - paperless-consumer.service + - paperless-scheduler.service + - paperless-webserver.service + register: paperless_services + +- name: reload systemd daemon + systemd: + name: "{{ item }}" + state: restarted + daemon_reload: yes + with_items: + - paperless-consumer + - paperless-scheduler + - paperless-webserver + when: paperless_services.changed + +- name: enable paperlessng services + systemd: + name: "{{ item }}" + enabled: yes + masked: no + state: started + with_items: + - paperless-consumer + - paperless-scheduler + - paperless-webserver From 093c4f1b9651c0ac50d9bb46c0c4f1b33e17c5e4 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 22 Dec 2020 19:30:53 +0100 Subject: [PATCH 02/20] Update to 0.9.9 --- ansible/defaults/main.yml | 5 +++-- ansible/tasks/main.yml | 14 ++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 0fa03b47a..fbeef9871 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.8 +paperlessng_version: 0.9.9 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" @@ -32,8 +32,9 @@ paperlessng_redis_host: localhost paperlessng_redis_port: 6379 paperlessng_db_type: sqlite # or postgresql +# Below entries only apply for paperlessng_db_type=='postgresql' paperlessng_db_host: localhost paperlessng_db_port: 5432 paperlessng_db_name: paperlessng paperlessng_db_user: paperlessng -paperlessng_db_pass: paperlessng \ No newline at end of file +paperlessng_db_pass: paperlessng diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index b427482b2..96b1e469b 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -169,7 +169,7 @@ line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - no_log: true + #no_log: true - name: configure paperless-ng database [sqlite] lineinfile: @@ -209,17 +209,21 @@ extra_args: --upgrade - name: collect static files - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --clear --no-input" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --clear --no-input" + args: + chdir: "{{ paperlessng_directory }}/src" - name: create database schema - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" + args: + chdir: "{{ paperlessng_directory }}/src" register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' - name: create first paperless user # "manage.py createsuperuser" only works on interactive TTYs command: | - {{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c " + {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " from django.contrib.auth.models import User from django.contrib.auth.hashers import get_hasher @@ -242,6 +246,8 @@ User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') print('changed') " + args: + chdir: "{{ paperlessng_directory }}/src" register: superuser changed_when: superuser.stdout == 'changed' no_log: true From f4e2c46d92dae55d3fba5d4feff3add01a1b38a0 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 13:30:09 +0100 Subject: [PATCH 03/20] Allow running role on all Debian releases Dynamically template current release string where required. --- ansible/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 96b1e469b..5b0bb5802 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: verify operating system fail: - msg: Sorry, only Debian 10 supported at the moment. - when: not(ansible_distribution == 'Debian' and ansible_distribution_version == '10') + msg: Sorry, only Debian supported at the moment. + when: ansible_distribution != 'Debian' - name: install base dependencies apt: @@ -45,7 +45,7 @@ - name: set up notesalexp repository (for jbig2enc) apt_repository: - repo: deb https://notesalexp.org/debian/buster/ buster main + repo: "deb https://notesalexp.org/debian/{{ ansible_distribution_release }}/ {{ ansible_distribution_release }} main" state: present when: paperlessng_use_jbig2enc @@ -169,7 +169,7 @@ line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - #no_log: true + no_log: true - name: configure paperless-ng database [sqlite] lineinfile: From 76e95d75f81ea3ef6140f423c5a31108016a9398 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 13:39:16 +0100 Subject: [PATCH 04/20] Fix fresh installation We can't backup a nonexistent folder. --- ansible/tasks/main.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 5b0bb5802..ba83b4975 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -81,6 +81,19 @@ state: started when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' +- name: create paperless group + group: + name: "{{ paperlessng_system_group }}" + +- name: create paperless user + user: + name: "{{ paperlessng_system_user }}" + groups: + - "{{ paperlessng_system_group }}" + shell: /usr/sbin/nologin + # GNUPG_HOME required due to paperless db.py + create_home: yes + - name: check for paperless-ng installation command: cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' @@ -94,7 +107,7 @@ src: "{{ paperlessng_directory }}" dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" remote_src: yes - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string' - name: download paperless-ng get_url: @@ -258,19 +271,6 @@ regexp: '' line: '' -- name: create paperless group - group: - name: "{{ paperlessng_system_group }}" - -- name: create paperless user - user: - name: "{{ paperlessng_system_user }}" - groups: - - "{{ paperlessng_system_group }}" - shell: /usr/sbin/nologin - # GNUPG_HOME required due to paperless db.py - create_home: yes - - name: configure systemd services ini_file: path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" From 36ac1eeb57b9314b2356218567c4c74652ed4009 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 14:05:35 +0100 Subject: [PATCH 05/20] Do not clear static files on every run Django collectstatic knows when files change. --- ansible/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index ba83b4975..9c8fd67f5 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -222,7 +222,7 @@ extra_args: --upgrade - name: collect static files - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --clear --no-input" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" args: chdir: "{{ paperlessng_directory }}/src" From 2b761cb3c67bfb7d23c404e465450d3216d999fe Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 09:47:17 +0100 Subject: [PATCH 06/20] Drop all permissions to paperlessng user Also make role idempotent --- ansible/tasks/main.yml | 105 ++++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 9c8fd67f5..77635b3d1 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -81,11 +81,11 @@ state: started when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' -- name: create paperless group +- name: create paperless system group group: name: "{{ paperlessng_system_group }}" -- name: create paperless user +- name: create paperless system user user: name: "{{ paperlessng_system_user }}" groups: @@ -105,31 +105,10 @@ - name: backup current paperless-ng installation copy: src: "{{ paperlessng_directory }}" - dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" remote_src: yes + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string' -- name: download paperless-ng - get_url: - url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" - dest: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - -- name: create paperless-ng directories - file: - path: "{{ item }}" - state: directory - owner: "{{ paperlessng_system_user }}" - group: "{{ paperlessng_system_group }}" - mode: 0750 - recurse: yes - with_items: - - "{{ paperlessng_directory }}" - - "{{ paperlessng_consumption_dir }}" - - "{{ paperlessng_data_dir }}" - - "{{ paperlessng_media_root }}" - - "{{ paperlessng_static_dir }}" - - name: create temporary directory tempfile: state: directory @@ -138,16 +117,28 @@ - name: extract paperless-ng unarchive: - src: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz - dest: "{{ tempdir.path }}" + src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" remote_src: yes + dest: "{{ tempdir.path }}" + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: change permissions of paperless-ng + command: + cmd: "{{ item }}" + with_items: + - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" + - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: move paperless-ng - command: - cmd: "cp -R {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" - args: - warn: false + copy: + src: "{{ tempdir.path }}/paperless-ng/" + remote_src: yes + dest: "{{ paperlessng_directory }}" + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: preserve + directory_mode: preserve when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary directory @@ -156,6 +147,20 @@ state: absent when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' +- name: create paperless-ng directories and set permissions + file: + path: "{{ item }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: "750" + with_items: + - "{{ paperlessng_directory }}" # ansible `copy:` does not set correct permissions on `dest:` for recursive copies + - "{{ paperlessng_consumption_dir }}" + - "{{ paperlessng_data_dir }}" + - "{{ paperlessng_media_root }}" + - "{{ paperlessng_static_dir }}" + - name: configure paperless-ng lineinfile: path: "{{ paperlessng_directory }}/paperless.conf" @@ -176,10 +181,10 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARG=" - # TODO JSON dict required in conf? - # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings - line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + # - regexp: "^#PAPERLESS_OCR_USER_ARG=" + # # TODO JSON dict required in conf + # # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings + # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: true @@ -211,29 +216,45 @@ no_log: true - name: create paperlessng venv + become: yes + become_user: "{{ paperlessng_system_user }}" command: cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3" creates: "{{ paperlessng_virtualenv }}" + register: venv - name: install paperlessng requirements + become: yes + become_user: "{{ paperlessng_system_user }}" pip: requirements: "{{ paperlessng_directory }}/requirements.txt" - virtualenv: "{{ paperlessng_virtualenv }}" + executable: "{{ paperlessng_virtualenv }}/bin/pip3" extra_args: --upgrade + when: paperlessng_current_version.stdout != paperlessng_version | string - name: collect static files + become: yes + become_user: "{{ paperlessng_system_user }}" command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" args: chdir: "{{ paperlessng_directory }}/src" + when: paperlessng_current_version.stdout != paperlessng_version | string + register: static_files + changed_when: "'188 unmodified' not in static_files.stdout" - name: create database schema + become: yes + become_user: "{{ paperlessng_system_user }}" command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" args: chdir: "{{ paperlessng_directory }}/src" + when: paperlessng_current_version.stdout != paperlessng_version | string register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' -- name: create first paperless user +- name: configure paperless superuser + become: yes + become_user: "{{ paperlessng_system_user }}" # "manage.py createsuperuser" only works on interactive TTYs command: | {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " @@ -265,6 +286,16 @@ changed_when: superuser.stdout == 'changed' no_log: true +- name: set ownership and permissions on paperlessng venv + file: + path: "{{ paperlessng_virtualenv }}" + state: directory + recurse: yes + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: g-w,o-rwx + when: venv.changed or paperlessng_current_version.stdout != paperlessng_version | string + - name: configure ghostscript for PDF lineinfile: path: "/etc/ImageMagick-6/policy.xml" @@ -325,8 +356,8 @@ - name: copy systemd services copy: src: "{{ paperlessng_directory }}/scripts/{{ item }}" - dest: "/etc/systemd/system/{{ item }}" remote_src: yes + dest: "/etc/systemd/system/{{ item }}" with_items: - paperless-consumer.service - paperless-scheduler.service From c9d90d8ae018b7d8fae961a355a9dee31a76c5ad Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 11:28:19 +0100 Subject: [PATCH 07/20] Add molecule test for role Only test default installation with jbig2enc and sqlite --- .travis.yml | 10 ++++++++++ ansible/molecule/default/converge.yml | 7 +++++++ ansible/molecule/default/molecule.yml | 21 +++++++++++++++++++++ ansible/molecule/default/verify.yml | 14 ++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 ansible/molecule/default/converge.yml create mode 100644 ansible/molecule/default/molecule.yml create mode 100644 ansible/molecule/default/verify.yml diff --git a/.travis.yml b/.travis.yml index b745d6bd7..e36b1ca99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,16 @@ jobs: - ng build --prod after_success: true + - name: "Ansible role" + sudo: required + services: + - docker + install: + - python3 -m pip install molecule[ansible,docker] + script: + - cd ansible/ + - molecule test + after_success: true before_install: - sudo apt-get update -qq diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml new file mode 100644 index 000000000..82067f417 --- /dev/null +++ b/ansible/molecule/default/converge.yml @@ -0,0 +1,7 @@ +--- +- name: Converge + hosts: all + tasks: + - name: "Include ansible" + include_role: + name: "ansible" diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml new file mode 100644 index 000000000..1bf938d98 --- /dev/null +++ b/ansible/molecule/default/molecule.yml @@ -0,0 +1,21 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: debian_buster + image: jrei/systemd-debian:10 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /tmp + - /run + - /run/lock + override_command: False + # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies +provisioner: + name: ansible +verifier: + name: ansible diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml new file mode 100644 index 000000000..4c3b86044 --- /dev/null +++ b/ansible/molecule/default/verify.yml @@ -0,0 +1,14 @@ +--- +# This is an example playbook to execute Ansible tests. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: check if webserver is up + uri: + url: http://localhost:8000 + status_code: [200, 302] + return_content: yes + register: landingpage + failed_when: "'Sign in' not in landingpage.content" From 39dc72b36dc30646c30326f401ecd9f4978bc20c Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 12:21:10 +0100 Subject: [PATCH 08/20] Verify role for Ubuntu 20.04 --- ansible/molecule/default/molecule.yml | 14 ++++++++++++++ ansible/tasks/main.yml | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml index 1bf938d98..27f37ba63 100644 --- a/ansible/molecule/default/molecule.yml +++ b/ansible/molecule/default/molecule.yml @@ -4,6 +4,20 @@ dependency: driver: name: docker platforms: + - name: ubuntu_focal + image: jrei/systemd-ubuntu:20.04 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /tmp + - /run + - /run/lock + override_command: False + # ubuntu 18.04 bionic works except that + # the default redis configuration expects IPv6 which is not enabled in docker by default + # the default Python environment is configured for ASCII instead of UTF-8 + # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies - name: debian_buster image: jrei/systemd-debian:10 privileged: true diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 77635b3d1..c94534895 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: verify operating system fail: - msg: Sorry, only Debian supported at the moment. - when: ansible_distribution != 'Debian' + msg: Sorry, only Debian and Ubuntu supported at the moment. + when: not(ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu') - name: install base dependencies apt: From dd3d55c6b94754395233df0e98f1e86b0ed6a27f Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 20:59:49 +0100 Subject: [PATCH 09/20] Make role compatible with ansible 2.7 Recursive remote copy is supported starting with 2.8 only Indentation behaviour in literal yaml strings seems to have changed Regex logic for ImageMagic was flawed (no idea why this worked before) --- ansible/tasks/main.yml | 69 +++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index c94534895..b1932a33c 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -28,6 +28,7 @@ - pngquant - zlib1g # dev + - sudo - build-essential - python3-setuptools - python3-wheel @@ -122,23 +123,19 @@ dest: "{{ tempdir.path }}" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' -- name: change permissions of paperless-ng +- name: change owner and permissions of paperless-ng command: cmd: "{{ item }}" + warn: false with_items: + - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}" - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: move paperless-ng - copy: - src: "{{ tempdir.path }}/paperless-ng/" - remote_src: yes - dest: "{{ paperlessng_directory }}" - owner: "{{ paperlessng_system_user }}" - group: "{{ paperlessng_system_group }}" - mode: preserve - directory_mode: preserve + command: + cmd: "cp -a {{ tempdir.path }}/paperless-ng/ {{ paperlessng_directory }}" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary directory @@ -256,30 +253,31 @@ become: yes become_user: "{{ paperlessng_system_user }}" # "manage.py createsuperuser" only works on interactive TTYs + vars: + creation_script: | + from django.contrib.auth.models import User + from django.contrib.auth.hashers import get_hasher + + if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): + user = User.objects.get(username='{{ paperlessng_superuser_name }}') + old = user.__dict__.copy() + + user.is_superuser = True + user.email = '{{ paperlessng_superuser_email }}' + user.set_password('{{ paperlessng_superuser_password }}') + user.save() + new = user.__dict__ + + algorithm, iterations, old_salt, old_hash = old['password'].split('$') + new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) + _, _, _, new_hash = new_password_old_salt.split('$') + if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): + print('changed') + else: + User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') + print('changed') command: | - {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " - from django.contrib.auth.models import User - from django.contrib.auth.hashers import get_hasher - - if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): - user = User.objects.get(username='{{ paperlessng_superuser_name }}') - old = user.__dict__.copy() - - user.is_superuser = True - user.email = '{{ paperlessng_superuser_email }}' - user.set_password('{{ paperlessng_superuser_password }}') - user.save() - new = user.__dict__ - - algorithm, iterations, old_salt, old_hash = old['password'].split('$') - new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) - _, _, _, new_hash = new_password_old_salt.split('$') - if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): - print('changed') - else: - User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') - print('changed') - " + {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c "{{ creation_script }}" args: chdir: "{{ paperlessng_directory }}/src" register: superuser @@ -298,9 +296,10 @@ - name: configure ghostscript for PDF lineinfile: - path: "/etc/ImageMagick-6/policy.xml" - regexp: '' - line: '' + path: /etc/ImageMagick-6/policy.xml + regexp: '(\s+)' + line: '\1' + backrefs: yes - name: configure systemd services ini_file: From f1bd6f1a4c3d95098bc4f27ee9af17f3accc5caf Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 21:55:59 +0100 Subject: [PATCH 10/20] Move paperless.conf to /etc, drop permissions --- ansible/tasks/main.yml | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index b1932a33c..2ee2e4db2 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -158,9 +158,14 @@ - "{{ paperlessng_media_root }}" - "{{ paperlessng_static_dir }}" +- name: rename initial config + command: + cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template" + removes: "{{ paperlessng_directory }}/paperless.conf" + - name: configure paperless-ng lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "{{ item.regexp }}" line: "{{ item.line }}" with_items: @@ -184,18 +189,18 @@ # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - no_log: true + no_log: yes - name: configure paperless-ng database [sqlite] lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "^#?PAPERLESS_DBHOST=" state: absent when: paperlessng_db_type == 'sqlite' - name: configure paperless-ng database [postgresql] lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "{{ item.regexp }}" line: "{{ item.line }}" with_items: @@ -210,7 +215,17 @@ - regexp: "^#?PAPERLESS_DBPASS=" line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" when: paperlessng_db_type == 'postgresql' - no_log: true + no_log: yes + +- name: deploy paperless-ng configuration + copy: + src: "{{ paperlessng_directory }}/paperless.conf.template" + remote_src: yes + dest: /etc/paperless.conf + owner: root + group: root + mode: '0644' + register: configuration - name: create paperlessng venv become: yes @@ -232,9 +247,7 @@ - name: collect static files become: yes become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --no-input" when: paperlessng_current_version.stdout != paperlessng_version | string register: static_files changed_when: "'188 unmodified' not in static_files.stdout" @@ -242,9 +255,7 @@ - name: create database schema become: yes become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" when: paperlessng_current_version.stdout != paperlessng_version | string register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' @@ -276,13 +287,10 @@ else: User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') print('changed') - command: | - {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c "{{ creation_script }}" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c \"{{ creation_script }}\"" register: superuser changed_when: superuser.stdout == 'changed' - no_log: true + no_log: yes - name: set ownership and permissions on paperlessng venv file: @@ -372,7 +380,7 @@ - paperless-consumer - paperless-scheduler - paperless-webserver - when: paperless_services.changed + when: paperless_services.changed or configuration.changed - name: enable paperlessng services systemd: From ce8c968c79152953dc046438050eab04421fdbcb Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 22:43:52 +0100 Subject: [PATCH 11/20] Integrate OCRmyPDF args into ansible config --- ansible/defaults/main.yml | 11 ++++++----- ansible/tasks/main.yml | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index fbeef9871..2504f18dc 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -11,12 +11,13 @@ paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv" paperlessng_ocr_languages: - eng paperlessng_time_zone: Europe/Berlin -paperlessng_ocrmypdf_args: --optimize 1 -# TODO Does optimze==1 really work with jbig2enc? -# https://ocrmypdf.readthedocs.io/en/latest/jbig2.html#lossy-mode-jbig2 -# Documentation states -O1 only applies lossless transformations -# https://ocrmypdf.readthedocs.io/en/latest/optimizer.html#lossless-optimizations +# see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr +paperlessng_ocrmypdf_args: + - "deskew": true + - "clean": true + - "optimize": 1 paperlessng_use_jbig2enc: true +paperlessng_big2enc_lossy: false paperlessng_superuser_name: paperlessng paperlessng_superuser_email: paperlessng@example.com diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 2ee2e4db2..4a243f322 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -183,10 +183,8 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - # - regexp: "^#PAPERLESS_OCR_USER_ARG=" - # # TODO JSON dict required in conf - # # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings - # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + - regexp: "^#PAPERLESS_OCR_USER_ARG=" + line: "PAPERLESS_OCR_USER_ARG={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) }}" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: yes From 4d076908908ae3da79e95a09cfc006b2e941e798 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 23:30:59 +0100 Subject: [PATCH 12/20] Harden systemd service files, drop perms further --- ansible/tasks/main.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 4a243f322..f0fd84d67 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -310,7 +310,7 @@ - name: configure systemd services ini_file: path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" - section: "{{ item[1].section }}" + section: "Service" option: "{{ item[1].option }}" value: "{{ item[1].value }}" with_nested: @@ -320,21 +320,35 @@ paperless-webserver.service, ] - [ + # https://www.freedesktop.org/software/systemd/man/systemd.exec.html { - section: "Service", option: "User", value: "{{ paperlessng_system_user }}", }, { - section: "Service", option: "Group", value: "{{ paperlessng_system_group }}", }, { - section: "Service", option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", }, + { + option: "ProtectSystem", + value: "full", + }, + { + option: "NoNewPrivileges", + value: "true", + }, + { + option: "PrivateUsers", + value: "true", + }, + { + option: "PrivateDevices", + value: "true", + } ] - name: configure paperless-consumer service From 6b16307636f361c0b45941ed1ad3b5ff5c898a65 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 23:54:22 +0100 Subject: [PATCH 13/20] Fix creation of user arg json.loads is picky in that is expects true json, not yaml from ansible --- ansible/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index f0fd84d67..604b29203 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -183,8 +183,8 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARG=" - line: "PAPERLESS_OCR_USER_ARG={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) }}" + - regexp: "^#PAPERLESS_OCR_USER_ARGS=" + line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: yes From 61e6819b33a719a0397814daa0379acae2e2ef47 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 09:53:29 +0100 Subject: [PATCH 14/20] Update to 0.9.10 --- ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 2504f18dc..adfdb2b87 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.9 +paperlessng_version: 0.9.10 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" From 0a66b9ff8c586a007028af099a9d89d24c6fc5bd Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 10:01:57 +0100 Subject: [PATCH 15/20] Change default OCRmyPDF args Clean is hardcoded as True, anyway Deskew breaks tesseract redo --- ansible/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index adfdb2b87..a2c560701 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -13,8 +13,7 @@ paperlessng_ocr_languages: paperlessng_time_zone: Europe/Berlin # see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr paperlessng_ocrmypdf_args: - - "deskew": true - - "clean": true + #- "deskew": true - "optimize": 1 paperlessng_use_jbig2enc: true paperlessng_big2enc_lossy: false From cbfd1062934d9597bab66cb612fb64cfda3d3e10 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 11:47:54 +0100 Subject: [PATCH 16/20] Update to 0.9.11 --- ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index a2c560701..da38ca300 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.10 +paperlessng_version: 0.9.11 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" From 93e69ebf383e37216f35ccc2687ca758ffeac9e0 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 11:48:05 +0100 Subject: [PATCH 17/20] Switch to github actions --- .github/workflows/ansible.yml | 33 +++++++++++++++++++++++++++++++++ .travis.yml | 10 ---------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/ansible.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml new file mode 100644 index 000000000..41c13c774 --- /dev/null +++ b/.github/workflows/ansible.yml @@ -0,0 +1,33 @@ +--- +name: Ansible Role + +on: push + +jobs: + test: + # https://molecule.readthedocs.io/en/latest/ci.html#github-actions + runs-on: ubuntu-latest + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: contains(github.ref, 'refs/heads/') + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Set up Docker + uses: docker-practice/actions-setup-docker@master + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install molecule[ansible,docker] + ansible --version + docker --version + molecule --version + python --version + - name: Test with molecule + run: | + cd ansible + molecule test + working-directory: "${{ github.repository }}" diff --git a/.travis.yml b/.travis.yml index e36b1ca99..b745d6bd7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,16 +33,6 @@ jobs: - ng build --prod after_success: true - - name: "Ansible role" - sudo: required - services: - - docker - install: - - python3 -m pip install molecule[ansible,docker] - script: - - cd ansible/ - - molecule test - after_success: true before_install: - sudo apt-get update -qq From b436102a724ad629e660367d2b37b9bb1b76a57a Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 13:21:42 +0100 Subject: [PATCH 18/20] Deploy role to ansible galaxy Happens only when a new tag is pushed --- .github/workflows/ansible.yml | 32 +++++++++++++++++++++++++---- ansible/README.md | 38 +++++++++++++++++++++++++++++++++++ ansible/meta/main.yml | 17 ++++++++++++++++ 3 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 ansible/README.md create mode 100644 ansible/meta/main.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 41c13c774..67383ecf8 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -1,14 +1,14 @@ --- name: Ansible Role -on: push +on: [push, pull_request] jobs: - test: # https://molecule.readthedocs.io/en/latest/ci.html#github-actions + test: runs-on: ubuntu-latest # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context - if: contains(github.ref, 'refs/heads/') + if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/')) steps: - name: Check out the codebase uses: actions/checkout@v2 @@ -16,7 +16,7 @@ jobs: path: "${{ github.repository }}" - name: Set up Python uses: actions/setup-python@v2 - - name: Set up Docker + - name: Set up Docker uses: docker-practice/actions-setup-docker@master - name: Install dependencies run: | @@ -31,3 +31,27 @@ jobs: cd ansible molecule test working-directory: "${{ github.repository }}" + # https://galaxy.ansible.com/docs/contributing/importing.html + release: + runs-on: ubuntu-latest + needs: + - test + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: contains(github.ref, 'refs/tags/') + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python3 -m pip install --upgrade ansible-base + ansible --version + python --version + - name: Trigger a new import on Galaxy + run: | + cd ansible + ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) + working-directory: "${{ github.repository }}" diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/ansible/meta/main.yml b/ansible/meta/main.yml new file mode 100644 index 000000000..78a4c1a41 --- /dev/null +++ b/ansible/meta/main.yml @@ -0,0 +1,17 @@ +dependencies: [] + +galaxy_info: + author: C0nsultant + description: Bare-metal deployment of paperless-ng DMS + license: license (GPLv3) + min_ansible_version: 2.7 + + platforms: + - name: Debian + versions: + - buster + - name: Ubuntu + versions: + - focal + + galaxy_tags: [EDMS, django, python, web] From 48013e87b96f7c96a2240183fe5da811ffc92e8a Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 9 Jan 2021 10:46:38 +0100 Subject: [PATCH 19/20] Disable ansible-galaxy import Repository structure not compatible (galaxy expects the role to be at the root of the repository, not in a subfolder). --- .github/workflows/ansible.yml | 50 ++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 67383ecf8..aacc4b44e 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -31,27 +31,29 @@ jobs: cd ansible molecule test working-directory: "${{ github.repository }}" - # https://galaxy.ansible.com/docs/contributing/importing.html - release: - runs-on: ubuntu-latest - needs: - - test - # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context - if: contains(github.ref, 'refs/tags/') - steps: - - name: Check out the codebase - uses: actions/checkout@v2 - with: - path: "${{ github.repository }}" - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade ansible-base - ansible --version - python --version - - name: Trigger a new import on Galaxy - run: | - cd ansible - ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) - working-directory: "${{ github.repository }}" + # # https://galaxy.ansible.com/docs/contributing/importing.html + # release: + # runs-on: ubuntu-latest + # needs: + # - test + # # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + # if: contains(github.ref, 'refs/tags/') + # steps: + # - name: Check out the codebase + # uses: actions/checkout@v2 + # with: + # path: "${{ github.repository }}" + # - name: Set up Python + # uses: actions/setup-python@v2 + # - name: Install dependencies + # run: | + # python3 -m pip install --upgrade ansible-base + # ansible --version + # python --version + # - name: Trigger a new import on Galaxy + # # TODO Check if source if pulled from cwd or imported from github + # # https://github.com/ansible/ansible/blob/devel/lib/ansible/cli/galaxy.py + # run: | + # cd ansible + # ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) + # working-directory: "${{ github.repository }}" From f0b2dbbe9454a024dee39eba7d37fb34edcbe3f2 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 9 Jan 2021 11:52:39 +0100 Subject: [PATCH 20/20] Update to 0.9.13 --- ansible/defaults/main.yml | 7 +++-- ansible/tasks/main.yml | 60 ++++++++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index da38ca300..d54bb855f 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.11 +paperlessng_version: 0.9.13 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" @@ -13,10 +13,13 @@ paperlessng_ocr_languages: paperlessng_time_zone: Europe/Berlin # see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr paperlessng_ocrmypdf_args: - #- "deskew": true + #- "deskew": true # https://github.com/jonaswinkler/paperless-ng/issues/231 - "optimize": 1 paperlessng_use_jbig2enc: true paperlessng_big2enc_lossy: false +paperlessng_tika_enabled: false +paperlessng_tika_endpoint: http://localhost:9998 +paperlessng_tika_gotenberg_endpoint: http://localhost:3000 paperlessng_superuser_name: paperlessng paperlessng_superuser_email: paperlessng@example.com diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 604b29203..c4ee8c875 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -11,6 +11,8 @@ # paperless-ng - python3-dev - python3-pip + - gettext + - fonts-liberation - imagemagick - unpaper - ghostscript @@ -166,51 +168,65 @@ - name: configure paperless-ng lineinfile: path: "{{ paperlessng_directory }}/paperless.conf.template" - regexp: "{{ item.regexp }}" + regexp: "^#?{{ item.regexp }}=" line: "{{ item.line }}" with_items: - - regexp: "^#?PAPERLESS_REDIS=" + - regexp: PAPERLESS_REDIS line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}" - - regexp: "^#?PAPERLESS_CONSUMPTION_DIR=" + - regexp: PAPERLESS_CONSUMPTION_DIR line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}" - - regexp: "^#?PAPERLESS_DATA_DIR=" + - regexp: PAPERLESS_DATA_DIR line: "PAPERLESS_DATA_DIR={{ paperlessng_data_dir }}" - - regexp: "^#?PAPERLESS_MEDIA_ROOT=" + - regexp: PAPERLESS_MEDIA_ROOT line: "PAPERLESS_MEDIA_ROOT={{ paperlessng_media_root }}" - - regexp: "^#?PAPERLESS_STATICDIR=" + - regexp: PAPERLESS_STATICDIR line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}" - - regexp: "^#?PAPERLESS_FILENAME_FORMAT=" + - regexp: PAPERLESS_FILENAME_FORMAT line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" + - regexp: PAPERLESS_OCR_LANGUAGE line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARGS=" + - regexp: PAPERLESS_OCR_USER_ARGS line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}" - - regexp: "^#?PAPERLESS_TIME_ZONE=" + - regexp: PAPERLESS_TIME_ZONE line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" + - regexp: PAPERLESS_TIKA_ENABLED + line: "PAPERLESS_TIKA_ENABLED={{ paperlessng_tika_enabled }}" no_log: yes +- name: configure paperless-ng [tika] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf.template" + regexp: "^#?{{ item.regexp }}=" + line: "'{{ item.line }}' if paperlessng_tika_enabled else '#{{ item.line }}'" + with_items: + - regexp: PAPERLESS_TIKA_ENDPOINT + line: "PAPERLESS_TIKA_ENDPOINT={{ paperlessng_tika_endpoint }}" + - regexp: PAPERLESS_TIKA_GOTENBERG_ENDPOINT + line: "PAPERLESS_TIKA_GOTENBERG_ENDPOINT={{ paperlessng_tika_endpoint }}" + - name: configure paperless-ng database [sqlite] lineinfile: path: "{{ paperlessng_directory }}/paperless.conf.template" - regexp: "^#?PAPERLESS_DBHOST=" - state: absent + regexp: "^#?PAPERLESS_DBHOST=(.*)$" + line: '#PAPERLESS_DBHOST=\1' + backrefs: yes when: paperlessng_db_type == 'sqlite' - name: configure paperless-ng database [postgresql] lineinfile: path: "{{ paperlessng_directory }}/paperless.conf.template" - regexp: "{{ item.regexp }}" + regexp: "^#?{{ item.regexp }}=" line: "{{ item.line }}" with_items: - - regexp: "^#?PAPERLESS_DBHOST=" + - regexp: PAPERLESS_DBHOST line: "PAPERLESS_DBHOST={{ paperlessng_db_host }}" - - regexp: "^#?PAPERLESS_DBPORT=" + - regexp: PAPERLESS_DBPORT line: "PAPERLESS_DBPORT={{ paperlessng_db_port }}" - - regexp: "^#?PAPERLESS_DBNAME=" + - regexp: PAPERLESS_DBNAME line: "PAPERLESS_DBNAME={{ paperlessng_db_name }}" - - regexp: "^#?PAPERLESS_DBUSER=" + - regexp: PAPERLESS_DBUSER line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}" - - regexp: "^#?PAPERLESS_DBPASS=" + - regexp: PAPERLESS_DBPASS line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" when: paperlessng_db_type == 'postgresql' no_log: yes @@ -248,7 +264,7 @@ command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --no-input" when: paperlessng_current_version.stdout != paperlessng_version | string register: static_files - changed_when: "'188 unmodified' not in static_files.stdout" + changed_when: static_files.stdout is not match("0 static files copied .*") - name: create database schema become: yes @@ -258,6 +274,12 @@ register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' +- name: compile translations + become: yes + become_user: "{{ paperlessng_system_user }}" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py compilemessages" + when: paperlessng_current_version.stdout != paperlessng_version | string + - name: configure paperless superuser become: yes become_user: "{{ paperlessng_system_user }}"