From 50c5a23de8ae1cd86f15260b459972b27d1a1d32 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 22 Dec 2020 18:46:47 +0100 Subject: [PATCH 01/20] add basic ansible role for debian deployment Currently only Debian 10 buster is supported. Other Debian versions, Ubuntu and derivates should be easy to integrate. Database deployment is considered out-of-scope and deferred to the user. Provides basic upgrade support between releases. --- ansible/defaults/main.yml | 39 +++++ ansible/tasks/main.yml | 350 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 389 insertions(+) create mode 100644 ansible/defaults/main.yml create mode 100644 ansible/tasks/main.yml diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml new file mode 100644 index 000000000..0fa03b47a --- /dev/null +++ b/ansible/defaults/main.yml @@ -0,0 +1,39 @@ +--- +paperlessng_version: 0.9.8 +paperlessng_directory: /opt/paperless-ng +paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" +paperlessng_data_dir: "{{ paperlessng_directory }}/data" +paperlessng_media_root: "{{ paperlessng_directory }}/media" +paperlessng_static_dir: "{{ paperlessng_directory }}/static" +paperlessng_filename_format: +paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv" + +paperlessng_ocr_languages: + - eng +paperlessng_time_zone: Europe/Berlin +paperlessng_ocrmypdf_args: --optimize 1 +# TODO Does optimze==1 really work with jbig2enc? +# https://ocrmypdf.readthedocs.io/en/latest/jbig2.html#lossy-mode-jbig2 +# Documentation states -O1 only applies lossless transformations +# https://ocrmypdf.readthedocs.io/en/latest/optimizer.html#lossless-optimizations +paperlessng_use_jbig2enc: true + +paperlessng_superuser_name: paperlessng +paperlessng_superuser_email: paperlessng@example.com +paperlessng_superuser_password: paperlessng + +paperlessng_system_user: paperlessng +paperlessng_system_group: paperlessng + +paperlessng_listen_address: 127.0.0.1 +paperlessng_listen_port: 8000 + +paperlessng_redis_host: localhost +paperlessng_redis_port: 6379 + +paperlessng_db_type: sqlite # or postgresql +paperlessng_db_host: localhost +paperlessng_db_port: 5432 +paperlessng_db_name: paperlessng +paperlessng_db_user: paperlessng +paperlessng_db_pass: paperlessng \ No newline at end of file diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml new file mode 100644 index 000000000..b427482b2 --- /dev/null +++ b/ansible/tasks/main.yml @@ -0,0 +1,350 @@ +--- +- name: verify operating system + fail: + msg: Sorry, only Debian 10 supported at the moment. + when: not(ansible_distribution == 'Debian' and ansible_distribution_version == '10') + +- name: install base dependencies + apt: + update_cache: yes + pkg: + # paperless-ng + - python3-dev + - python3-pip + - imagemagick + - unpaper + - ghostscript + - optipng + - tesseract-ocr + - gnupg + - libpoppler-cpp-dev + - libmagic-dev + - libpq-dev + # OCRmyPDF + - icc-profiles-free + - qpdf + - liblept5 + - libxml2 + - pngquant + - zlib1g + # dev + - build-essential + - python3-setuptools + - python3-wheel + - python3-virtualenv + +- name: install ocr languages + apt: + pkg: "{{ paperlessng_ocr_languages | map('regex_replace', '^(.*)$', 'tesseract-ocr-\\1') | list }}" + +- name: set up notesalexp repository key (for jbig2enc) + apt_key: + url: https://notesalexp.org/debian/alexp_key.asc + state: present + when: paperlessng_use_jbig2enc + +- name: set up notesalexp repository (for jbig2enc) + apt_repository: + repo: deb https://notesalexp.org/debian/buster/ buster main + state: present + when: paperlessng_use_jbig2enc + +- name: set up notesalexp repository pinning + copy: + content: | + Package: * + Pin: release o=notesalexp.org + Pin-Priority: 1 + + Package: jbig2enc + Pin: release o=notesalexp.org + Pin-Priority: 500 + dest: /etc/apt/preferences.d/notesalexp + when: paperlessng_use_jbig2enc + +- name: install jbig2enc + apt: + pkg: jbig2enc + update_cache: yes + when: paperlessng_use_jbig2enc + +- name: install redis + apt: + pkg: redis-server + when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' + +- name: enable redis + systemd: + name: redis-server + enabled: yes + masked: no + state: started + when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' + +- name: check for paperless-ng installation + command: + cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' + changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + failed_when: false + ignore_errors: yes + register: paperlessng_current_version + +- name: backup current paperless-ng installation + copy: + src: "{{ paperlessng_directory }}" + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" + remote_src: yes + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: download paperless-ng + get_url: + url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + dest: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: create paperless-ng directories + file: + path: "{{ item }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: 0750 + recurse: yes + with_items: + - "{{ paperlessng_directory }}" + - "{{ paperlessng_consumption_dir }}" + - "{{ paperlessng_data_dir }}" + - "{{ paperlessng_media_root }}" + - "{{ paperlessng_static_dir }}" + +- name: create temporary directory + tempfile: + state: directory + register: tempdir + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: extract paperless-ng + unarchive: + src: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz + dest: "{{ tempdir.path }}" + remote_src: yes + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: move paperless-ng + command: + cmd: "cp -R {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" + args: + warn: false + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: remove temporary directory + file: + path: "{{ tempdir.path }}" + state: absent + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: configure paperless-ng + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - regexp: "^#?PAPERLESS_REDIS=" + line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}" + - regexp: "^#?PAPERLESS_CONSUMPTION_DIR=" + line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}" + - regexp: "^#?PAPERLESS_DATA_DIR=" + line: "PAPERLESS_DATA_DIR={{ paperlessng_data_dir }}" + - regexp: "^#?PAPERLESS_MEDIA_ROOT=" + line: "PAPERLESS_MEDIA_ROOT={{ paperlessng_media_root }}" + - regexp: "^#?PAPERLESS_STATICDIR=" + line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}" + - regexp: "^#?PAPERLESS_FILENAME_FORMAT=" + line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" + - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" + line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" + - regexp: "^#PAPERLESS_OCR_USER_ARG=" + # TODO JSON dict required in conf? + # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings + line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + - regexp: "^#?PAPERLESS_TIME_ZONE=" + line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" + no_log: true + +- name: configure paperless-ng database [sqlite] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "^#?PAPERLESS_DBHOST=" + state: absent + when: paperlessng_db_type == 'sqlite' + +- name: configure paperless-ng database [postgresql] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - regexp: "^#?PAPERLESS_DBHOST=" + line: "PAPERLESS_DBHOST={{ paperlessng_db_host }}" + - regexp: "^#?PAPERLESS_DBPORT=" + line: "PAPERLESS_DBPORT={{ paperlessng_db_port }}" + - regexp: "^#?PAPERLESS_DBNAME=" + line: "PAPERLESS_DBNAME={{ paperlessng_db_name }}" + - regexp: "^#?PAPERLESS_DBUSER=" + line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}" + - regexp: "^#?PAPERLESS_DBPASS=" + line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" + when: paperlessng_db_type == 'postgresql' + no_log: true + +- name: create paperlessng venv + command: + cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3" + creates: "{{ paperlessng_virtualenv }}" + +- name: install paperlessng requirements + pip: + requirements: "{{ paperlessng_directory }}/requirements.txt" + virtualenv: "{{ paperlessng_virtualenv }}" + extra_args: --upgrade + +- name: collect static files + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --clear --no-input" + +- name: create database schema + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + register: database_schema + changed_when: '"No migrations to apply." not in database_schema.stdout' + +- name: create first paperless user + # "manage.py createsuperuser" only works on interactive TTYs + command: | + {{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c " + from django.contrib.auth.models import User + from django.contrib.auth.hashers import get_hasher + + if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): + user = User.objects.get(username='{{ paperlessng_superuser_name }}') + old = user.__dict__.copy() + + user.is_superuser = True + user.email = '{{ paperlessng_superuser_email }}' + user.set_password('{{ paperlessng_superuser_password }}') + user.save() + new = user.__dict__ + + algorithm, iterations, old_salt, old_hash = old['password'].split('$') + new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) + _, _, _, new_hash = new_password_old_salt.split('$') + if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): + print('changed') + else: + User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') + print('changed') + " + register: superuser + changed_when: superuser.stdout == 'changed' + no_log: true + +- name: configure ghostscript for PDF + lineinfile: + path: "/etc/ImageMagick-6/policy.xml" + regexp: '' + line: '' + +- name: create paperless group + group: + name: "{{ paperlessng_system_group }}" + +- name: create paperless user + user: + name: "{{ paperlessng_system_user }}" + groups: + - "{{ paperlessng_system_group }}" + shell: /usr/sbin/nologin + # GNUPG_HOME required due to paperless db.py + create_home: yes + +- name: configure systemd services + ini_file: + path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" + section: "{{ item[1].section }}" + option: "{{ item[1].option }}" + value: "{{ item[1].value }}" + with_nested: + - [ + paperless-consumer.service, + paperless-scheduler.service, + paperless-webserver.service, + ] + - [ + { + section: "Service", + option: "User", + value: "{{ paperlessng_system_user }}", + }, + { + section: "Service", + option: "Group", + value: "{{ paperlessng_system_group }}", + }, + { + section: "Service", + option: "WorkingDirectory", + value: "{{ paperlessng_directory }}/src", + }, + ] + +- name: configure paperless-consumer service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-consumer.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py document_consumer" + +- name: configure paperless-scheduler service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-scheduler.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py qcluster" + +- name: configure paperless-webserver service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}" + +- name: copy systemd services + copy: + src: "{{ paperlessng_directory }}/scripts/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + remote_src: yes + with_items: + - paperless-consumer.service + - paperless-scheduler.service + - paperless-webserver.service + register: paperless_services + +- name: reload systemd daemon + systemd: + name: "{{ item }}" + state: restarted + daemon_reload: yes + with_items: + - paperless-consumer + - paperless-scheduler + - paperless-webserver + when: paperless_services.changed + +- name: enable paperlessng services + systemd: + name: "{{ item }}" + enabled: yes + masked: no + state: started + with_items: + - paperless-consumer + - paperless-scheduler + - paperless-webserver From e48294a74b18283ce4c60288eed907a1c0c8dbb1 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 22 Dec 2020 19:30:53 +0100 Subject: [PATCH 02/20] Update to 0.9.9 --- ansible/defaults/main.yml | 5 +++-- ansible/tasks/main.yml | 14 ++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 0fa03b47a..fbeef9871 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.8 +paperlessng_version: 0.9.9 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" @@ -32,8 +32,9 @@ paperlessng_redis_host: localhost paperlessng_redis_port: 6379 paperlessng_db_type: sqlite # or postgresql +# Below entries only apply for paperlessng_db_type=='postgresql' paperlessng_db_host: localhost paperlessng_db_port: 5432 paperlessng_db_name: paperlessng paperlessng_db_user: paperlessng -paperlessng_db_pass: paperlessng \ No newline at end of file +paperlessng_db_pass: paperlessng diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index b427482b2..96b1e469b 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -169,7 +169,7 @@ line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - no_log: true + #no_log: true - name: configure paperless-ng database [sqlite] lineinfile: @@ -209,17 +209,21 @@ extra_args: --upgrade - name: collect static files - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --clear --no-input" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --clear --no-input" + args: + chdir: "{{ paperlessng_directory }}/src" - name: create database schema - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" + args: + chdir: "{{ paperlessng_directory }}/src" register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' - name: create first paperless user # "manage.py createsuperuser" only works on interactive TTYs command: | - {{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c " + {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " from django.contrib.auth.models import User from django.contrib.auth.hashers import get_hasher @@ -242,6 +246,8 @@ User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') print('changed') " + args: + chdir: "{{ paperlessng_directory }}/src" register: superuser changed_when: superuser.stdout == 'changed' no_log: true From 92fa97873543cbd499ceaca8787765f231027071 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 13:30:09 +0100 Subject: [PATCH 03/20] Allow running role on all Debian releases Dynamically template current release string where required. --- ansible/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 96b1e469b..5b0bb5802 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: verify operating system fail: - msg: Sorry, only Debian 10 supported at the moment. - when: not(ansible_distribution == 'Debian' and ansible_distribution_version == '10') + msg: Sorry, only Debian supported at the moment. + when: ansible_distribution != 'Debian' - name: install base dependencies apt: @@ -45,7 +45,7 @@ - name: set up notesalexp repository (for jbig2enc) apt_repository: - repo: deb https://notesalexp.org/debian/buster/ buster main + repo: "deb https://notesalexp.org/debian/{{ ansible_distribution_release }}/ {{ ansible_distribution_release }} main" state: present when: paperlessng_use_jbig2enc @@ -169,7 +169,7 @@ line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - #no_log: true + no_log: true - name: configure paperless-ng database [sqlite] lineinfile: From 089a8c0498c1e3264209284a7f5804c093b7d3d2 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 13:39:16 +0100 Subject: [PATCH 04/20] Fix fresh installation We can't backup a nonexistent folder. --- ansible/tasks/main.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 5b0bb5802..ba83b4975 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -81,6 +81,19 @@ state: started when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' +- name: create paperless group + group: + name: "{{ paperlessng_system_group }}" + +- name: create paperless user + user: + name: "{{ paperlessng_system_user }}" + groups: + - "{{ paperlessng_system_group }}" + shell: /usr/sbin/nologin + # GNUPG_HOME required due to paperless db.py + create_home: yes + - name: check for paperless-ng installation command: cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' @@ -94,7 +107,7 @@ src: "{{ paperlessng_directory }}" dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" remote_src: yes - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string' - name: download paperless-ng get_url: @@ -258,19 +271,6 @@ regexp: '' line: '' -- name: create paperless group - group: - name: "{{ paperlessng_system_group }}" - -- name: create paperless user - user: - name: "{{ paperlessng_system_user }}" - groups: - - "{{ paperlessng_system_group }}" - shell: /usr/sbin/nologin - # GNUPG_HOME required due to paperless db.py - create_home: yes - - name: configure systemd services ini_file: path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" From 227934a7f0941e76c8c6b51f552aea25ab2e3222 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 14:05:35 +0100 Subject: [PATCH 05/20] Do not clear static files on every run Django collectstatic knows when files change. --- ansible/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index ba83b4975..9c8fd67f5 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -222,7 +222,7 @@ extra_args: --upgrade - name: collect static files - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --clear --no-input" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" args: chdir: "{{ paperlessng_directory }}/src" From ef9631ae241889cfca5ce7f18b71d2575ae5a673 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 09:47:17 +0100 Subject: [PATCH 06/20] Drop all permissions to paperlessng user Also make role idempotent --- ansible/tasks/main.yml | 105 ++++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 9c8fd67f5..77635b3d1 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -81,11 +81,11 @@ state: started when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' -- name: create paperless group +- name: create paperless system group group: name: "{{ paperlessng_system_group }}" -- name: create paperless user +- name: create paperless system user user: name: "{{ paperlessng_system_user }}" groups: @@ -105,31 +105,10 @@ - name: backup current paperless-ng installation copy: src: "{{ paperlessng_directory }}" - dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" remote_src: yes + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string' -- name: download paperless-ng - get_url: - url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" - dest: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - -- name: create paperless-ng directories - file: - path: "{{ item }}" - state: directory - owner: "{{ paperlessng_system_user }}" - group: "{{ paperlessng_system_group }}" - mode: 0750 - recurse: yes - with_items: - - "{{ paperlessng_directory }}" - - "{{ paperlessng_consumption_dir }}" - - "{{ paperlessng_data_dir }}" - - "{{ paperlessng_media_root }}" - - "{{ paperlessng_static_dir }}" - - name: create temporary directory tempfile: state: directory @@ -138,16 +117,28 @@ - name: extract paperless-ng unarchive: - src: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz - dest: "{{ tempdir.path }}" + src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" remote_src: yes + dest: "{{ tempdir.path }}" + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: change permissions of paperless-ng + command: + cmd: "{{ item }}" + with_items: + - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" + - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: move paperless-ng - command: - cmd: "cp -R {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" - args: - warn: false + copy: + src: "{{ tempdir.path }}/paperless-ng/" + remote_src: yes + dest: "{{ paperlessng_directory }}" + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: preserve + directory_mode: preserve when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary directory @@ -156,6 +147,20 @@ state: absent when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' +- name: create paperless-ng directories and set permissions + file: + path: "{{ item }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: "750" + with_items: + - "{{ paperlessng_directory }}" # ansible `copy:` does not set correct permissions on `dest:` for recursive copies + - "{{ paperlessng_consumption_dir }}" + - "{{ paperlessng_data_dir }}" + - "{{ paperlessng_media_root }}" + - "{{ paperlessng_static_dir }}" + - name: configure paperless-ng lineinfile: path: "{{ paperlessng_directory }}/paperless.conf" @@ -176,10 +181,10 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARG=" - # TODO JSON dict required in conf? - # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings - line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + # - regexp: "^#PAPERLESS_OCR_USER_ARG=" + # # TODO JSON dict required in conf + # # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings + # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: true @@ -211,29 +216,45 @@ no_log: true - name: create paperlessng venv + become: yes + become_user: "{{ paperlessng_system_user }}" command: cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3" creates: "{{ paperlessng_virtualenv }}" + register: venv - name: install paperlessng requirements + become: yes + become_user: "{{ paperlessng_system_user }}" pip: requirements: "{{ paperlessng_directory }}/requirements.txt" - virtualenv: "{{ paperlessng_virtualenv }}" + executable: "{{ paperlessng_virtualenv }}/bin/pip3" extra_args: --upgrade + when: paperlessng_current_version.stdout != paperlessng_version | string - name: collect static files + become: yes + become_user: "{{ paperlessng_system_user }}" command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" args: chdir: "{{ paperlessng_directory }}/src" + when: paperlessng_current_version.stdout != paperlessng_version | string + register: static_files + changed_when: "'188 unmodified' not in static_files.stdout" - name: create database schema + become: yes + become_user: "{{ paperlessng_system_user }}" command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" args: chdir: "{{ paperlessng_directory }}/src" + when: paperlessng_current_version.stdout != paperlessng_version | string register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' -- name: create first paperless user +- name: configure paperless superuser + become: yes + become_user: "{{ paperlessng_system_user }}" # "manage.py createsuperuser" only works on interactive TTYs command: | {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " @@ -265,6 +286,16 @@ changed_when: superuser.stdout == 'changed' no_log: true +- name: set ownership and permissions on paperlessng venv + file: + path: "{{ paperlessng_virtualenv }}" + state: directory + recurse: yes + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: g-w,o-rwx + when: venv.changed or paperlessng_current_version.stdout != paperlessng_version | string + - name: configure ghostscript for PDF lineinfile: path: "/etc/ImageMagick-6/policy.xml" @@ -325,8 +356,8 @@ - name: copy systemd services copy: src: "{{ paperlessng_directory }}/scripts/{{ item }}" - dest: "/etc/systemd/system/{{ item }}" remote_src: yes + dest: "/etc/systemd/system/{{ item }}" with_items: - paperless-consumer.service - paperless-scheduler.service From 1276419ec6e6691280d98bfb30081f926166e6e5 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 11:28:19 +0100 Subject: [PATCH 07/20] Add molecule test for role Only test default installation with jbig2enc and sqlite --- .travis.yml | 10 ++++++++++ ansible/molecule/default/converge.yml | 7 +++++++ ansible/molecule/default/molecule.yml | 21 +++++++++++++++++++++ ansible/molecule/default/verify.yml | 14 ++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 ansible/molecule/default/converge.yml create mode 100644 ansible/molecule/default/molecule.yml create mode 100644 ansible/molecule/default/verify.yml diff --git a/.travis.yml b/.travis.yml index b745d6bd7..e36b1ca99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,16 @@ jobs: - ng build --prod after_success: true + - name: "Ansible role" + sudo: required + services: + - docker + install: + - python3 -m pip install molecule[ansible,docker] + script: + - cd ansible/ + - molecule test + after_success: true before_install: - sudo apt-get update -qq diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml new file mode 100644 index 000000000..82067f417 --- /dev/null +++ b/ansible/molecule/default/converge.yml @@ -0,0 +1,7 @@ +--- +- name: Converge + hosts: all + tasks: + - name: "Include ansible" + include_role: + name: "ansible" diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml new file mode 100644 index 000000000..1bf938d98 --- /dev/null +++ b/ansible/molecule/default/molecule.yml @@ -0,0 +1,21 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: debian_buster + image: jrei/systemd-debian:10 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /tmp + - /run + - /run/lock + override_command: False + # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies +provisioner: + name: ansible +verifier: + name: ansible diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml new file mode 100644 index 000000000..4c3b86044 --- /dev/null +++ b/ansible/molecule/default/verify.yml @@ -0,0 +1,14 @@ +--- +# This is an example playbook to execute Ansible tests. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: check if webserver is up + uri: + url: http://localhost:8000 + status_code: [200, 302] + return_content: yes + register: landingpage + failed_when: "'Sign in' not in landingpage.content" From bdf2e298435b2e3139224f91a880842de3193eae Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 12:21:10 +0100 Subject: [PATCH 08/20] Verify role for Ubuntu 20.04 --- ansible/molecule/default/molecule.yml | 14 ++++++++++++++ ansible/tasks/main.yml | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml index 1bf938d98..27f37ba63 100644 --- a/ansible/molecule/default/molecule.yml +++ b/ansible/molecule/default/molecule.yml @@ -4,6 +4,20 @@ dependency: driver: name: docker platforms: + - name: ubuntu_focal + image: jrei/systemd-ubuntu:20.04 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /tmp + - /run + - /run/lock + override_command: False + # ubuntu 18.04 bionic works except that + # the default redis configuration expects IPv6 which is not enabled in docker by default + # the default Python environment is configured for ASCII instead of UTF-8 + # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies - name: debian_buster image: jrei/systemd-debian:10 privileged: true diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 77635b3d1..c94534895 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: verify operating system fail: - msg: Sorry, only Debian supported at the moment. - when: ansible_distribution != 'Debian' + msg: Sorry, only Debian and Ubuntu supported at the moment. + when: not(ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu') - name: install base dependencies apt: From bf3ffc29a9aee03a160612032fd78f31226732bb Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 20:59:49 +0100 Subject: [PATCH 09/20] Make role compatible with ansible 2.7 Recursive remote copy is supported starting with 2.8 only Indentation behaviour in literal yaml strings seems to have changed Regex logic for ImageMagic was flawed (no idea why this worked before) --- ansible/tasks/main.yml | 69 +++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index c94534895..b1932a33c 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -28,6 +28,7 @@ - pngquant - zlib1g # dev + - sudo - build-essential - python3-setuptools - python3-wheel @@ -122,23 +123,19 @@ dest: "{{ tempdir.path }}" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' -- name: change permissions of paperless-ng +- name: change owner and permissions of paperless-ng command: cmd: "{{ item }}" + warn: false with_items: + - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}" - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: move paperless-ng - copy: - src: "{{ tempdir.path }}/paperless-ng/" - remote_src: yes - dest: "{{ paperlessng_directory }}" - owner: "{{ paperlessng_system_user }}" - group: "{{ paperlessng_system_group }}" - mode: preserve - directory_mode: preserve + command: + cmd: "cp -a {{ tempdir.path }}/paperless-ng/ {{ paperlessng_directory }}" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary directory @@ -256,30 +253,31 @@ become: yes become_user: "{{ paperlessng_system_user }}" # "manage.py createsuperuser" only works on interactive TTYs + vars: + creation_script: | + from django.contrib.auth.models import User + from django.contrib.auth.hashers import get_hasher + + if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): + user = User.objects.get(username='{{ paperlessng_superuser_name }}') + old = user.__dict__.copy() + + user.is_superuser = True + user.email = '{{ paperlessng_superuser_email }}' + user.set_password('{{ paperlessng_superuser_password }}') + user.save() + new = user.__dict__ + + algorithm, iterations, old_salt, old_hash = old['password'].split('$') + new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) + _, _, _, new_hash = new_password_old_salt.split('$') + if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): + print('changed') + else: + User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') + print('changed') command: | - {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " - from django.contrib.auth.models import User - from django.contrib.auth.hashers import get_hasher - - if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): - user = User.objects.get(username='{{ paperlessng_superuser_name }}') - old = user.__dict__.copy() - - user.is_superuser = True - user.email = '{{ paperlessng_superuser_email }}' - user.set_password('{{ paperlessng_superuser_password }}') - user.save() - new = user.__dict__ - - algorithm, iterations, old_salt, old_hash = old['password'].split('$') - new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) - _, _, _, new_hash = new_password_old_salt.split('$') - if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): - print('changed') - else: - User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') - print('changed') - " + {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c "{{ creation_script }}" args: chdir: "{{ paperlessng_directory }}/src" register: superuser @@ -298,9 +296,10 @@ - name: configure ghostscript for PDF lineinfile: - path: "/etc/ImageMagick-6/policy.xml" - regexp: '' - line: '' + path: /etc/ImageMagick-6/policy.xml + regexp: '(\s+)' + line: '\1' + backrefs: yes - name: configure systemd services ini_file: From f075384b449aa390926dae1c8f0e4e4f7e411146 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 21:55:59 +0100 Subject: [PATCH 10/20] Move paperless.conf to /etc, drop permissions --- ansible/tasks/main.yml | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index b1932a33c..2ee2e4db2 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -158,9 +158,14 @@ - "{{ paperlessng_media_root }}" - "{{ paperlessng_static_dir }}" +- name: rename initial config + command: + cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template" + removes: "{{ paperlessng_directory }}/paperless.conf" + - name: configure paperless-ng lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "{{ item.regexp }}" line: "{{ item.line }}" with_items: @@ -184,18 +189,18 @@ # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - no_log: true + no_log: yes - name: configure paperless-ng database [sqlite] lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "^#?PAPERLESS_DBHOST=" state: absent when: paperlessng_db_type == 'sqlite' - name: configure paperless-ng database [postgresql] lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "{{ item.regexp }}" line: "{{ item.line }}" with_items: @@ -210,7 +215,17 @@ - regexp: "^#?PAPERLESS_DBPASS=" line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" when: paperlessng_db_type == 'postgresql' - no_log: true + no_log: yes + +- name: deploy paperless-ng configuration + copy: + src: "{{ paperlessng_directory }}/paperless.conf.template" + remote_src: yes + dest: /etc/paperless.conf + owner: root + group: root + mode: '0644' + register: configuration - name: create paperlessng venv become: yes @@ -232,9 +247,7 @@ - name: collect static files become: yes become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --no-input" when: paperlessng_current_version.stdout != paperlessng_version | string register: static_files changed_when: "'188 unmodified' not in static_files.stdout" @@ -242,9 +255,7 @@ - name: create database schema become: yes become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" when: paperlessng_current_version.stdout != paperlessng_version | string register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' @@ -276,13 +287,10 @@ else: User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') print('changed') - command: | - {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c "{{ creation_script }}" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c \"{{ creation_script }}\"" register: superuser changed_when: superuser.stdout == 'changed' - no_log: true + no_log: yes - name: set ownership and permissions on paperlessng venv file: @@ -372,7 +380,7 @@ - paperless-consumer - paperless-scheduler - paperless-webserver - when: paperless_services.changed + when: paperless_services.changed or configuration.changed - name: enable paperlessng services systemd: From bb569b4e7875298049efe3e4c9b4cf29018267f9 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 22:43:52 +0100 Subject: [PATCH 11/20] Integrate OCRmyPDF args into ansible config --- ansible/defaults/main.yml | 11 ++++++----- ansible/tasks/main.yml | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index fbeef9871..2504f18dc 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -11,12 +11,13 @@ paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv" paperlessng_ocr_languages: - eng paperlessng_time_zone: Europe/Berlin -paperlessng_ocrmypdf_args: --optimize 1 -# TODO Does optimze==1 really work with jbig2enc? -# https://ocrmypdf.readthedocs.io/en/latest/jbig2.html#lossy-mode-jbig2 -# Documentation states -O1 only applies lossless transformations -# https://ocrmypdf.readthedocs.io/en/latest/optimizer.html#lossless-optimizations +# see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr +paperlessng_ocrmypdf_args: + - "deskew": true + - "clean": true + - "optimize": 1 paperlessng_use_jbig2enc: true +paperlessng_big2enc_lossy: false paperlessng_superuser_name: paperlessng paperlessng_superuser_email: paperlessng@example.com diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 2ee2e4db2..4a243f322 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -183,10 +183,8 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - # - regexp: "^#PAPERLESS_OCR_USER_ARG=" - # # TODO JSON dict required in conf - # # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings - # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + - regexp: "^#PAPERLESS_OCR_USER_ARG=" + line: "PAPERLESS_OCR_USER_ARG={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) }}" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: yes From 14f87f5aee52753ada73ffcee8ab1f110e25af4c Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 23:30:59 +0100 Subject: [PATCH 12/20] Harden systemd service files, drop perms further --- ansible/tasks/main.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 4a243f322..f0fd84d67 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -310,7 +310,7 @@ - name: configure systemd services ini_file: path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" - section: "{{ item[1].section }}" + section: "Service" option: "{{ item[1].option }}" value: "{{ item[1].value }}" with_nested: @@ -320,21 +320,35 @@ paperless-webserver.service, ] - [ + # https://www.freedesktop.org/software/systemd/man/systemd.exec.html { - section: "Service", option: "User", value: "{{ paperlessng_system_user }}", }, { - section: "Service", option: "Group", value: "{{ paperlessng_system_group }}", }, { - section: "Service", option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", }, + { + option: "ProtectSystem", + value: "full", + }, + { + option: "NoNewPrivileges", + value: "true", + }, + { + option: "PrivateUsers", + value: "true", + }, + { + option: "PrivateDevices", + value: "true", + } ] - name: configure paperless-consumer service From e4e4efcba7e287d7133ea194d5785d88ff937963 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 23:54:22 +0100 Subject: [PATCH 13/20] Fix creation of user arg json.loads is picky in that is expects true json, not yaml from ansible --- ansible/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index f0fd84d67..604b29203 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -183,8 +183,8 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARG=" - line: "PAPERLESS_OCR_USER_ARG={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) }}" + - regexp: "^#PAPERLESS_OCR_USER_ARGS=" + line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: yes From a74404170ddd2a9cd66d0e46c58b041ca9841bb4 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 09:53:29 +0100 Subject: [PATCH 14/20] Update to 0.9.10 --- ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 2504f18dc..adfdb2b87 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.9 +paperlessng_version: 0.9.10 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" From be56707a8a508953d605d4f6bb554ef27646dba1 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 10:01:57 +0100 Subject: [PATCH 15/20] Change default OCRmyPDF args Clean is hardcoded as True, anyway Deskew breaks tesseract redo --- ansible/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index adfdb2b87..a2c560701 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -13,8 +13,7 @@ paperlessng_ocr_languages: paperlessng_time_zone: Europe/Berlin # see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr paperlessng_ocrmypdf_args: - - "deskew": true - - "clean": true + #- "deskew": true - "optimize": 1 paperlessng_use_jbig2enc: true paperlessng_big2enc_lossy: false From fc31195fa0bd307e2a640dce7ef98cbc6e3df7fb Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 11:47:54 +0100 Subject: [PATCH 16/20] Update to 0.9.11 --- ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index a2c560701..da38ca300 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.10 +paperlessng_version: 0.9.11 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" From 51fd19c3156b5056db8478ea525e5f2d07255631 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 11:48:05 +0100 Subject: [PATCH 17/20] Switch to github actions --- .github/workflows/ansible.yml | 33 +++++++++++++++++++++++++++++++++ .travis.yml | 10 ---------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/ansible.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml new file mode 100644 index 000000000..41c13c774 --- /dev/null +++ b/.github/workflows/ansible.yml @@ -0,0 +1,33 @@ +--- +name: Ansible Role + +on: push + +jobs: + test: + # https://molecule.readthedocs.io/en/latest/ci.html#github-actions + runs-on: ubuntu-latest + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: contains(github.ref, 'refs/heads/') + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Set up Docker + uses: docker-practice/actions-setup-docker@master + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install molecule[ansible,docker] + ansible --version + docker --version + molecule --version + python --version + - name: Test with molecule + run: | + cd ansible + molecule test + working-directory: "${{ github.repository }}" diff --git a/.travis.yml b/.travis.yml index e36b1ca99..b745d6bd7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,16 +33,6 @@ jobs: - ng build --prod after_success: true - - name: "Ansible role" - sudo: required - services: - - docker - install: - - python3 -m pip install molecule[ansible,docker] - script: - - cd ansible/ - - molecule test - after_success: true before_install: - sudo apt-get update -qq From 586db3bc5e9bdd7962721f8bc0efee2af1e178a2 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 13:21:42 +0100 Subject: [PATCH 18/20] Deploy role to ansible galaxy Happens only when a new tag is pushed --- .github/workflows/ansible.yml | 32 +++++++++++++++++++++++++---- ansible/README.md | 38 +++++++++++++++++++++++++++++++++++ ansible/meta/main.yml | 17 ++++++++++++++++ 3 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 ansible/README.md create mode 100644 ansible/meta/main.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 41c13c774..67383ecf8 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -1,14 +1,14 @@ --- name: Ansible Role -on: push +on: [push, pull_request] jobs: - test: # https://molecule.readthedocs.io/en/latest/ci.html#github-actions + test: runs-on: ubuntu-latest # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context - if: contains(github.ref, 'refs/heads/') + if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/')) steps: - name: Check out the codebase uses: actions/checkout@v2 @@ -16,7 +16,7 @@ jobs: path: "${{ github.repository }}" - name: Set up Python uses: actions/setup-python@v2 - - name: Set up Docker + - name: Set up Docker uses: docker-practice/actions-setup-docker@master - name: Install dependencies run: | @@ -31,3 +31,27 @@ jobs: cd ansible molecule test working-directory: "${{ github.repository }}" + # https://galaxy.ansible.com/docs/contributing/importing.html + release: + runs-on: ubuntu-latest + needs: + - test + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: contains(github.ref, 'refs/tags/') + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python3 -m pip install --upgrade ansible-base + ansible --version + python --version + - name: Trigger a new import on Galaxy + run: | + cd ansible + ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) + working-directory: "${{ github.repository }}" diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/ansible/meta/main.yml b/ansible/meta/main.yml new file mode 100644 index 000000000..78a4c1a41 --- /dev/null +++ b/ansible/meta/main.yml @@ -0,0 +1,17 @@ +dependencies: [] + +galaxy_info: + author: C0nsultant + description: Bare-metal deployment of paperless-ng DMS + license: license (GPLv3) + min_ansible_version: 2.7 + + platforms: + - name: Debian + versions: + - buster + - name: Ubuntu + versions: + - focal + + galaxy_tags: [EDMS, django, python, web] From 9cd6235947e3b8a0483ef0dd6b611efcff206f17 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 9 Jan 2021 10:46:38 +0100 Subject: [PATCH 19/20] Disable ansible-galaxy import Repository structure not compatible (galaxy expects the role to be at the root of the repository, not in a subfolder). --- .github/workflows/ansible.yml | 50 ++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 67383ecf8..aacc4b44e 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -31,27 +31,29 @@ jobs: cd ansible molecule test working-directory: "${{ github.repository }}" - # https://galaxy.ansible.com/docs/contributing/importing.html - release: - runs-on: ubuntu-latest - needs: - - test - # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context - if: contains(github.ref, 'refs/tags/') - steps: - - name: Check out the codebase - uses: actions/checkout@v2 - with: - path: "${{ github.repository }}" - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade ansible-base - ansible --version - python --version - - name: Trigger a new import on Galaxy - run: | - cd ansible - ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) - working-directory: "${{ github.repository }}" + # # https://galaxy.ansible.com/docs/contributing/importing.html + # release: + # runs-on: ubuntu-latest + # needs: + # - test + # # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + # if: contains(github.ref, 'refs/tags/') + # steps: + # - name: Check out the codebase + # uses: actions/checkout@v2 + # with: + # path: "${{ github.repository }}" + # - name: Set up Python + # uses: actions/setup-python@v2 + # - name: Install dependencies + # run: | + # python3 -m pip install --upgrade ansible-base + # ansible --version + # python --version + # - name: Trigger a new import on Galaxy + # # TODO Check if source if pulled from cwd or imported from github + # # https://github.com/ansible/ansible/blob/devel/lib/ansible/cli/galaxy.py + # run: | + # cd ansible + # ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) + # working-directory: "${{ github.repository }}" From 69f1931f4fdba71c7157d38d8dd84544487d725d Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 9 Jan 2021 11:52:39 +0100 Subject: [PATCH 20/20] Update to 0.9.13 --- ansible/defaults/main.yml | 7 +++-- ansible/tasks/main.yml | 60 ++++++++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index da38ca300..d54bb855f 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.11 +paperlessng_version: 0.9.13 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" @@ -13,10 +13,13 @@ paperlessng_ocr_languages: paperlessng_time_zone: Europe/Berlin # see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr paperlessng_ocrmypdf_args: - #- "deskew": true + #- "deskew": true # https://github.com/jonaswinkler/paperless-ng/issues/231 - "optimize": 1 paperlessng_use_jbig2enc: true paperlessng_big2enc_lossy: false +paperlessng_tika_enabled: false +paperlessng_tika_endpoint: http://localhost:9998 +paperlessng_tika_gotenberg_endpoint: http://localhost:3000 paperlessng_superuser_name: paperlessng paperlessng_superuser_email: paperlessng@example.com diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 604b29203..c4ee8c875 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -11,6 +11,8 @@ # paperless-ng - python3-dev - python3-pip + - gettext + - fonts-liberation - imagemagick - unpaper - ghostscript @@ -166,51 +168,65 @@ - name: configure paperless-ng lineinfile: path: "{{ paperlessng_directory }}/paperless.conf.template" - regexp: "{{ item.regexp }}" + regexp: "^#?{{ item.regexp }}=" line: "{{ item.line }}" with_items: - - regexp: "^#?PAPERLESS_REDIS=" + - regexp: PAPERLESS_REDIS line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}" - - regexp: "^#?PAPERLESS_CONSUMPTION_DIR=" + - regexp: PAPERLESS_CONSUMPTION_DIR line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}" - - regexp: "^#?PAPERLESS_DATA_DIR=" + - regexp: PAPERLESS_DATA_DIR line: "PAPERLESS_DATA_DIR={{ paperlessng_data_dir }}" - - regexp: "^#?PAPERLESS_MEDIA_ROOT=" + - regexp: PAPERLESS_MEDIA_ROOT line: "PAPERLESS_MEDIA_ROOT={{ paperlessng_media_root }}" - - regexp: "^#?PAPERLESS_STATICDIR=" + - regexp: PAPERLESS_STATICDIR line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}" - - regexp: "^#?PAPERLESS_FILENAME_FORMAT=" + - regexp: PAPERLESS_FILENAME_FORMAT line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" + - regexp: PAPERLESS_OCR_LANGUAGE line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARGS=" + - regexp: PAPERLESS_OCR_USER_ARGS line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}" - - regexp: "^#?PAPERLESS_TIME_ZONE=" + - regexp: PAPERLESS_TIME_ZONE line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" + - regexp: PAPERLESS_TIKA_ENABLED + line: "PAPERLESS_TIKA_ENABLED={{ paperlessng_tika_enabled }}" no_log: yes +- name: configure paperless-ng [tika] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf.template" + regexp: "^#?{{ item.regexp }}=" + line: "'{{ item.line }}' if paperlessng_tika_enabled else '#{{ item.line }}'" + with_items: + - regexp: PAPERLESS_TIKA_ENDPOINT + line: "PAPERLESS_TIKA_ENDPOINT={{ paperlessng_tika_endpoint }}" + - regexp: PAPERLESS_TIKA_GOTENBERG_ENDPOINT + line: "PAPERLESS_TIKA_GOTENBERG_ENDPOINT={{ paperlessng_tika_endpoint }}" + - name: configure paperless-ng database [sqlite] lineinfile: path: "{{ paperlessng_directory }}/paperless.conf.template" - regexp: "^#?PAPERLESS_DBHOST=" - state: absent + regexp: "^#?PAPERLESS_DBHOST=(.*)$" + line: '#PAPERLESS_DBHOST=\1' + backrefs: yes when: paperlessng_db_type == 'sqlite' - name: configure paperless-ng database [postgresql] lineinfile: path: "{{ paperlessng_directory }}/paperless.conf.template" - regexp: "{{ item.regexp }}" + regexp: "^#?{{ item.regexp }}=" line: "{{ item.line }}" with_items: - - regexp: "^#?PAPERLESS_DBHOST=" + - regexp: PAPERLESS_DBHOST line: "PAPERLESS_DBHOST={{ paperlessng_db_host }}" - - regexp: "^#?PAPERLESS_DBPORT=" + - regexp: PAPERLESS_DBPORT line: "PAPERLESS_DBPORT={{ paperlessng_db_port }}" - - regexp: "^#?PAPERLESS_DBNAME=" + - regexp: PAPERLESS_DBNAME line: "PAPERLESS_DBNAME={{ paperlessng_db_name }}" - - regexp: "^#?PAPERLESS_DBUSER=" + - regexp: PAPERLESS_DBUSER line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}" - - regexp: "^#?PAPERLESS_DBPASS=" + - regexp: PAPERLESS_DBPASS line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" when: paperlessng_db_type == 'postgresql' no_log: yes @@ -248,7 +264,7 @@ command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --no-input" when: paperlessng_current_version.stdout != paperlessng_version | string register: static_files - changed_when: "'188 unmodified' not in static_files.stdout" + changed_when: static_files.stdout is not match("0 static files copied .*") - name: create database schema become: yes @@ -258,6 +274,12 @@ register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' +- name: compile translations + become: yes + become_user: "{{ paperlessng_system_user }}" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py compilemessages" + when: paperlessng_current_version.stdout != paperlessng_version | string + - name: configure paperless superuser become: yes become_user: "{{ paperlessng_system_user }}"