Merge branch 'master' into dev

This commit is contained in:
jonaswinkler 2021-01-12 14:14:06 +01:00
commit 775ce56d2e
8 changed files with 641 additions and 0 deletions

59
.github/workflows/ansible.yml vendored Normal file
View File

@ -0,0 +1,59 @@
---
name: Ansible Role
on: [push, pull_request]
jobs:
# https://molecule.readthedocs.io/en/latest/ci.html#github-actions
test:
runs-on: ubuntu-latest
# https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context
if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/'))
steps:
- name: Check out the codebase
uses: actions/checkout@v2
with:
path: "${{ github.repository }}"
- name: Set up Python
uses: actions/setup-python@v2
- name: Set up Docker
uses: docker-practice/actions-setup-docker@master
- name: Install dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install molecule[ansible,docker]
ansible --version
docker --version
molecule --version
python --version
- name: Test with molecule
run: |
cd ansible
molecule test
working-directory: "${{ github.repository }}"
# # https://galaxy.ansible.com/docs/contributing/importing.html
# release:
# runs-on: ubuntu-latest
# needs:
# - test
# # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context
# if: contains(github.ref, 'refs/tags/')
# steps:
# - name: Check out the codebase
# uses: actions/checkout@v2
# with:
# path: "${{ github.repository }}"
# - name: Set up Python
# uses: actions/setup-python@v2
# - name: Install dependencies
# run: |
# python3 -m pip install --upgrade ansible-base
# ansible --version
# python --version
# - name: Trigger a new import on Galaxy
# # TODO Check if source if pulled from cwd or imported from github
# # https://github.com/ansible/ansible/blob/devel/lib/ansible/cli/galaxy.py
# run: |
# cd ansible
# ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2)
# working-directory: "${{ github.repository }}"

38
ansible/README.md Normal file
View File

@ -0,0 +1,38 @@
Role Name
=========
A brief description of the role goes here.
Requirements
------------
Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
Role Variables
--------------
A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
Dependencies
------------
A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
Example Playbook
----------------
Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
- hosts: servers
roles:
- { role: username.rolename, x: 42 }
License
-------
BSD
Author Information
------------------
An optional section for the role authors to include contact information, or a website (HTML is not allowed).

43
ansible/defaults/main.yml Normal file
View File

@ -0,0 +1,43 @@
---
paperlessng_version: 0.9.13
paperlessng_directory: /opt/paperless-ng
paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption"
paperlessng_data_dir: "{{ paperlessng_directory }}/data"
paperlessng_media_root: "{{ paperlessng_directory }}/media"
paperlessng_static_dir: "{{ paperlessng_directory }}/static"
paperlessng_filename_format:
paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv"
paperlessng_ocr_languages:
- eng
paperlessng_time_zone: Europe/Berlin
# see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr
paperlessng_ocrmypdf_args:
#- "deskew": true # https://github.com/jonaswinkler/paperless-ng/issues/231
- "optimize": 1
paperlessng_use_jbig2enc: true
paperlessng_big2enc_lossy: false
paperlessng_tika_enabled: false
paperlessng_tika_endpoint: http://localhost:9998
paperlessng_tika_gotenberg_endpoint: http://localhost:3000
paperlessng_superuser_name: paperlessng
paperlessng_superuser_email: paperlessng@example.com
paperlessng_superuser_password: paperlessng
paperlessng_system_user: paperlessng
paperlessng_system_group: paperlessng
paperlessng_listen_address: 127.0.0.1
paperlessng_listen_port: 8000
paperlessng_redis_host: localhost
paperlessng_redis_port: 6379
paperlessng_db_type: sqlite # or postgresql
# Below entries only apply for paperlessng_db_type=='postgresql'
paperlessng_db_host: localhost
paperlessng_db_port: 5432
paperlessng_db_name: paperlessng
paperlessng_db_user: paperlessng
paperlessng_db_pass: paperlessng

17
ansible/meta/main.yml Normal file
View File

@ -0,0 +1,17 @@
dependencies: []
galaxy_info:
author: C0nsultant
description: Bare-metal deployment of paperless-ng DMS
license: license (GPLv3)
min_ansible_version: 2.7
platforms:
- name: Debian
versions:
- buster
- name: Ubuntu
versions:
- focal
galaxy_tags: [EDMS, django, python, web]

View File

@ -0,0 +1,7 @@
---
- name: Converge
hosts: all
tasks:
- name: "Include ansible"
include_role:
name: "ansible"

View File

@ -0,0 +1,35 @@
---
dependency:
name: galaxy
driver:
name: docker
platforms:
- name: ubuntu_focal
image: jrei/systemd-ubuntu:20.04
privileged: true
volumes:
- /sys/fs/cgroup:/sys/fs/cgroup:ro
tmpfs:
- /tmp
- /run
- /run/lock
override_command: False
# ubuntu 18.04 bionic works except that
# the default redis configuration expects IPv6 which is not enabled in docker by default
# the default Python environment is configured for ASCII instead of UTF-8
# ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies
- name: debian_buster
image: jrei/systemd-debian:10
privileged: true
volumes:
- /sys/fs/cgroup:/sys/fs/cgroup:ro
tmpfs:
- /tmp
- /run
- /run/lock
override_command: False
# debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies
provisioner:
name: ansible
verifier:
name: ansible

View File

@ -0,0 +1,14 @@
---
# This is an example playbook to execute Ansible tests.
- name: Verify
hosts: all
gather_facts: false
tasks:
- name: check if webserver is up
uri:
url: http://localhost:8000
status_code: [200, 302]
return_content: yes
register: landingpage
failed_when: "'Sign in</button>' not in landingpage.content"

428
ansible/tasks/main.yml Normal file
View File

@ -0,0 +1,428 @@
---
- name: verify operating system
fail:
msg: Sorry, only Debian and Ubuntu supported at the moment.
when: not(ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu')
- name: install base dependencies
apt:
update_cache: yes
pkg:
# paperless-ng
- python3-dev
- python3-pip
- gettext
- fonts-liberation
- imagemagick
- unpaper
- ghostscript
- optipng
- tesseract-ocr
- gnupg
- libpoppler-cpp-dev
- libmagic-dev
- libpq-dev
# OCRmyPDF
- icc-profiles-free
- qpdf
- liblept5
- libxml2
- pngquant
- zlib1g
# dev
- sudo
- build-essential
- python3-setuptools
- python3-wheel
- python3-virtualenv
- name: install ocr languages
apt:
pkg: "{{ paperlessng_ocr_languages | map('regex_replace', '^(.*)$', 'tesseract-ocr-\\1') | list }}"
- name: set up notesalexp repository key (for jbig2enc)
apt_key:
url: https://notesalexp.org/debian/alexp_key.asc
state: present
when: paperlessng_use_jbig2enc
- name: set up notesalexp repository (for jbig2enc)
apt_repository:
repo: "deb https://notesalexp.org/debian/{{ ansible_distribution_release }}/ {{ ansible_distribution_release }} main"
state: present
when: paperlessng_use_jbig2enc
- name: set up notesalexp repository pinning
copy:
content: |
Package: *
Pin: release o=notesalexp.org
Pin-Priority: 1
Package: jbig2enc
Pin: release o=notesalexp.org
Pin-Priority: 500
dest: /etc/apt/preferences.d/notesalexp
when: paperlessng_use_jbig2enc
- name: install jbig2enc
apt:
pkg: jbig2enc
update_cache: yes
when: paperlessng_use_jbig2enc
- name: install redis
apt:
pkg: redis-server
when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1'
- name: enable redis
systemd:
name: redis-server
enabled: yes
masked: no
state: started
when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1'
- name: create paperless system group
group:
name: "{{ paperlessng_system_group }}"
- name: create paperless system user
user:
name: "{{ paperlessng_system_user }}"
groups:
- "{{ paperlessng_system_group }}"
shell: /usr/sbin/nologin
# GNUPG_HOME required due to paperless db.py
create_home: yes
- name: check for paperless-ng installation
command:
cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html'
changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
failed_when: false
ignore_errors: yes
register: paperlessng_current_version
- name: backup current paperless-ng installation
copy:
src: "{{ paperlessng_directory }}"
remote_src: yes
dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/"
when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string'
- name: create temporary directory
tempfile:
state: directory
register: tempdir
when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
- name: extract paperless-ng
unarchive:
src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
remote_src: yes
dest: "{{ tempdir.path }}"
when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
- name: change owner and permissions of paperless-ng
command:
cmd: "{{ item }}"
warn: false
with_items:
- "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
- "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
- "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
- name: move paperless-ng
command:
cmd: "cp -a {{ tempdir.path }}/paperless-ng/ {{ paperlessng_directory }}"
when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
- name: remove temporary directory
file:
path: "{{ tempdir.path }}"
state: absent
when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
- name: create paperless-ng directories and set permissions
file:
path: "{{ item }}"
state: directory
owner: "{{ paperlessng_system_user }}"
group: "{{ paperlessng_system_group }}"
mode: "750"
with_items:
- "{{ paperlessng_directory }}" # ansible `copy:` does not set correct permissions on `dest:` for recursive copies
- "{{ paperlessng_consumption_dir }}"
- "{{ paperlessng_data_dir }}"
- "{{ paperlessng_media_root }}"
- "{{ paperlessng_static_dir }}"
- name: rename initial config
command:
cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template"
removes: "{{ paperlessng_directory }}/paperless.conf"
- name: configure paperless-ng
lineinfile:
path: "{{ paperlessng_directory }}/paperless.conf.template"
regexp: "^#?{{ item.regexp }}="
line: "{{ item.line }}"
with_items:
- regexp: PAPERLESS_REDIS
line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}"
- regexp: PAPERLESS_CONSUMPTION_DIR
line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}"
- regexp: PAPERLESS_DATA_DIR
line: "PAPERLESS_DATA_DIR={{ paperlessng_data_dir }}"
- regexp: PAPERLESS_MEDIA_ROOT
line: "PAPERLESS_MEDIA_ROOT={{ paperlessng_media_root }}"
- regexp: PAPERLESS_STATICDIR
line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}"
- regexp: PAPERLESS_FILENAME_FORMAT
line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}"
- regexp: PAPERLESS_OCR_LANGUAGE
line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}"
- regexp: PAPERLESS_OCR_USER_ARGS
line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}"
- regexp: PAPERLESS_TIME_ZONE
line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}"
- regexp: PAPERLESS_TIKA_ENABLED
line: "PAPERLESS_TIKA_ENABLED={{ paperlessng_tika_enabled }}"
no_log: yes
- name: configure paperless-ng [tika]
lineinfile:
path: "{{ paperlessng_directory }}/paperless.conf.template"
regexp: "^#?{{ item.regexp }}="
line: "'{{ item.line }}' if paperlessng_tika_enabled else '#{{ item.line }}'"
with_items:
- regexp: PAPERLESS_TIKA_ENDPOINT
line: "PAPERLESS_TIKA_ENDPOINT={{ paperlessng_tika_endpoint }}"
- regexp: PAPERLESS_TIKA_GOTENBERG_ENDPOINT
line: "PAPERLESS_TIKA_GOTENBERG_ENDPOINT={{ paperlessng_tika_endpoint }}"
- name: configure paperless-ng database [sqlite]
lineinfile:
path: "{{ paperlessng_directory }}/paperless.conf.template"
regexp: "^#?PAPERLESS_DBHOST=(.*)$"
line: '#PAPERLESS_DBHOST=\1'
backrefs: yes
when: paperlessng_db_type == 'sqlite'
- name: configure paperless-ng database [postgresql]
lineinfile:
path: "{{ paperlessng_directory }}/paperless.conf.template"
regexp: "^#?{{ item.regexp }}="
line: "{{ item.line }}"
with_items:
- regexp: PAPERLESS_DBHOST
line: "PAPERLESS_DBHOST={{ paperlessng_db_host }}"
- regexp: PAPERLESS_DBPORT
line: "PAPERLESS_DBPORT={{ paperlessng_db_port }}"
- regexp: PAPERLESS_DBNAME
line: "PAPERLESS_DBNAME={{ paperlessng_db_name }}"
- regexp: PAPERLESS_DBUSER
line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}"
- regexp: PAPERLESS_DBPASS
line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}"
when: paperlessng_db_type == 'postgresql'
no_log: yes
- name: deploy paperless-ng configuration
copy:
src: "{{ paperlessng_directory }}/paperless.conf.template"
remote_src: yes
dest: /etc/paperless.conf
owner: root
group: root
mode: '0644'
register: configuration
- name: create paperlessng venv
become: yes
become_user: "{{ paperlessng_system_user }}"
command:
cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3"
creates: "{{ paperlessng_virtualenv }}"
register: venv
- name: install paperlessng requirements
become: yes
become_user: "{{ paperlessng_system_user }}"
pip:
requirements: "{{ paperlessng_directory }}/requirements.txt"
executable: "{{ paperlessng_virtualenv }}/bin/pip3"
extra_args: --upgrade
when: paperlessng_current_version.stdout != paperlessng_version | string
- name: collect static files
become: yes
become_user: "{{ paperlessng_system_user }}"
command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --no-input"
when: paperlessng_current_version.stdout != paperlessng_version | string
register: static_files
changed_when: static_files.stdout is not match("0 static files copied .*")
- name: create database schema
become: yes
become_user: "{{ paperlessng_system_user }}"
command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate"
when: paperlessng_current_version.stdout != paperlessng_version | string
register: database_schema
changed_when: '"No migrations to apply." not in database_schema.stdout'
- name: compile translations
become: yes
become_user: "{{ paperlessng_system_user }}"
command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py compilemessages"
when: paperlessng_current_version.stdout != paperlessng_version | string
- name: configure paperless superuser
become: yes
become_user: "{{ paperlessng_system_user }}"
# "manage.py createsuperuser" only works on interactive TTYs
vars:
creation_script: |
from django.contrib.auth.models import User
from django.contrib.auth.hashers import get_hasher
if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists():
user = User.objects.get(username='{{ paperlessng_superuser_name }}')
old = user.__dict__.copy()
user.is_superuser = True
user.email = '{{ paperlessng_superuser_email }}'
user.set_password('{{ paperlessng_superuser_password }}')
user.save()
new = user.__dict__
algorithm, iterations, old_salt, old_hash = old['password'].split('$')
new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations))
_, _, _, new_hash = new_password_old_salt.split('$')
if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']):
print('changed')
else:
User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}')
print('changed')
command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c \"{{ creation_script }}\""
register: superuser
changed_when: superuser.stdout == 'changed'
no_log: yes
- name: set ownership and permissions on paperlessng venv
file:
path: "{{ paperlessng_virtualenv }}"
state: directory
recurse: yes
owner: "{{ paperlessng_system_user }}"
group: "{{ paperlessng_system_group }}"
mode: g-w,o-rwx
when: venv.changed or paperlessng_current_version.stdout != paperlessng_version | string
- name: configure ghostscript for PDF
lineinfile:
path: /etc/ImageMagick-6/policy.xml
regexp: '(\s+)<policy domain="coder" rights=".*" pattern="PDF" />'
line: '\1<policy domain="coder" rights="read|write" pattern="PDF" />'
backrefs: yes
- name: configure systemd services
ini_file:
path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}"
section: "Service"
option: "{{ item[1].option }}"
value: "{{ item[1].value }}"
with_nested:
- [
paperless-consumer.service,
paperless-scheduler.service,
paperless-webserver.service,
]
- [
# https://www.freedesktop.org/software/systemd/man/systemd.exec.html
{
option: "User",
value: "{{ paperlessng_system_user }}",
},
{
option: "Group",
value: "{{ paperlessng_system_group }}",
},
{
option: "WorkingDirectory",
value: "{{ paperlessng_directory }}/src",
},
{
option: "ProtectSystem",
value: "full",
},
{
option: "NoNewPrivileges",
value: "true",
},
{
option: "PrivateUsers",
value: "true",
},
{
option: "PrivateDevices",
value: "true",
}
]
- name: configure paperless-consumer service
ini_file:
path: "{{ paperlessng_directory }}/scripts/paperless-consumer.service"
section: "Service"
option: "ExecStart"
value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py document_consumer"
- name: configure paperless-scheduler service
ini_file:
path: "{{ paperlessng_directory }}/scripts/paperless-scheduler.service"
section: "Service"
option: "ExecStart"
value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py qcluster"
- name: configure paperless-webserver service
ini_file:
path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service"
section: "Service"
option: "ExecStart"
value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}"
- name: copy systemd services
copy:
src: "{{ paperlessng_directory }}/scripts/{{ item }}"
remote_src: yes
dest: "/etc/systemd/system/{{ item }}"
with_items:
- paperless-consumer.service
- paperless-scheduler.service
- paperless-webserver.service
register: paperless_services
- name: reload systemd daemon
systemd:
name: "{{ item }}"
state: restarted
daemon_reload: yes
with_items:
- paperless-consumer
- paperless-scheduler
- paperless-webserver
when: paperless_services.changed or configuration.changed
- name: enable paperlessng services
systemd:
name: "{{ item }}"
enabled: yes
masked: no
state: started
with_items:
- paperless-consumer
- paperless-scheduler
- paperless-webserver