Merge pull request #411 from C0nsultant/ansible-fromsource

Ansible - build release archive from source
This commit is contained in:
Jonas Winkler 2021-01-30 16:05:51 +01:00 committed by GitHub
commit ac459f7b05
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 372 additions and 249 deletions

View File

@ -21,20 +21,20 @@ jobs:
- name: Install dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install molecule[ansible,docker]
python3 -m pip install molecule[ansible,docker] jmespath
ansible --version
docker --version
molecule --version
python --version
- name: Test fresh installation with molecule
- name: Test installation/build/upgrade with molecule
run: |
cd ansible
molecule test -s fresh
working-directory: "${{ github.repository }}"
- name: Test release update with molecule
run: |
cd ansible
molecule test -s update
molecule create
molecule verify
molecule converge
molecule idempotence
molecule verify
molecule destroy
working-directory: "${{ github.repository }}"
# # https://galaxy.ansible.com/docs/contributing/importing.html
# release:

View File

@ -1,5 +1,5 @@
---
paperlessng_version: 0.9.14
paperlessng_version: latest # 'latest', release number, or github branch/tag/commit/ref
# Required services
paperlessng_redis_host: localhost

View File

@ -2,10 +2,9 @@
- name: update previous release to newest release
hosts: all
tasks:
- name: set current version as installation target
- name: set github ref as version when available
set_fact:
paperlessng_version: 0.9.14
paperlessng_version: "{{ lookup('env', 'GITHUB_REF') | default('latest', True) }}"
- name: update to newest paperless-ng release
include_role:
name: ansible

View File

@ -3,7 +3,7 @@
tasks:
- name: set previous version as installation target
set_fact:
paperlessng_version: 0.9.13
paperlessng_version: latest
- name: install previous paperless-ng release
include_role:

View File

@ -0,0 +1,94 @@
---
- name: Verify
hosts: all
gather_facts: false
vars_files:
- ../../defaults/main.yml
tasks:
- name: check if webserver is up
uri:
url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}"
status_code: [200, 302]
return_content: yes
register: landingpage
failed_when: "'Sign in</button>' not in landingpage.content"
- name: generate random name and content
set_fact:
content: "{{ lookup('password', '/dev/null length=65536 chars=ascii_letters') }}"
filename: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters') }}"
- name: check if document posting works
uri:
url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/post_document/"
method: POST
body_format: form-multipart
body:
document:
content: "{{ content }}"
filename: "{{ filename }}.txt"
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
Content-Type: text/plain
return_content: yes
register: post_document
failed_when: "'OK' not in post_document.content"
- name: verify uploaded document has been accepted
uri:
url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/"
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: logs
failed_when: "('Consuming ' + filename + '.txt') not in logs.content"
- name: sleep till consumption finished
pause:
seconds: 10
- name: verify uploaded document has been consumed
uri:
url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/"
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: logs
failed_when: "filename + ' consumption finished' not in logs.content"
- name: get documents
uri:
url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/"
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: documents
- name: set document index
set_fact:
index: "{{ documents.json['results'][0]['id'] }}"
- name: verify uploaded document is avaiable
uri:
url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/{{ index }}/"
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: document
failed_when: "'Not found.' in document.content or content not in document.json['content']"
- name: check if deleting uploaded document works
uri:
url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/bulk_edit/"
method: POST
body_format: json
body:
documents: ["{{ index }}"]
method: delete
parameters: {}
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
register: delete_document
failed_when: "'OK' not in delete_document.json['result']"

View File

@ -1,7 +0,0 @@
---
- name: fresh installation
hosts: all
tasks:
- name: install paperless-ng with default parameters
include_role:
name: ansible

View File

@ -1,60 +0,0 @@
---
- name: Verify
hosts: all
gather_facts: false
vars_files:
- ../../defaults/main.yml
tasks:
- name: check if webserver is up
uri:
url: http://localhost:8000
status_code: [200, 302]
return_content: yes
register: landingpage
failed_when: "'Sign in</button>' not in landingpage.content"
- name: check if document posting works
uri:
url: http://localhost:8000/api/documents/post_document/
method: POST
body_format: form-multipart
body:
document:
content: FOO
filename: document.txt
mime_type: text/plain
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: post_document
failed_when: "'OK' not in post_document.content"
- name: verify uploaded document has been accepted
uri:
url: http://localhost:8000/api/logs/
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: logs
failed_when: "'Consuming document.txt' not in logs.content"
# assumes txt consumption finished by now, might have to sleep a bit
- name: verify uploaded document has been consumed
uri:
url: http://localhost:8000/api/logs/
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: logs
failed_when: "'document consumption finished' not in logs.content"
- name: verify uploaded document is avaiable
uri:
url: http://localhost:8000/api/documents/1/
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: document
failed_when: "'Not found.' in document.content or 'FOO' not in document.content"

View File

@ -1,35 +0,0 @@
---
dependency:
name: galaxy
driver:
name: docker
platforms:
- name: ubuntu_focal
image: jrei/systemd-ubuntu:20.04
privileged: true
volumes:
- /sys/fs/cgroup:/sys/fs/cgroup:ro
tmpfs:
- /tmp
- /run
- /run/lock
override_command: False
# ubuntu 18.04 bionic works except that
# the default redis configuration expects IPv6 which is not enabled in docker by default
# the default Python environment is configured for ASCII instead of UTF-8
# ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies
- name: debian_buster
image: jrei/systemd-debian:10
privileged: true
volumes:
- /sys/fs/cgroup:/sys/fs/cgroup:ro
tmpfs:
- /tmp
- /run
- /run/lock
override_command: False
# debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies
provisioner:
name: ansible
verifier:
name: ansible

View File

@ -1,60 +0,0 @@
---
- name: Verify
hosts: all
gather_facts: false
vars_files:
- ../../defaults/main.yml
tasks:
- name: check if webserver is up
uri:
url: http://localhost:8000
status_code: [200, 302]
return_content: yes
register: landingpage
failed_when: "'Sign in</button>' not in landingpage.content"
- name: check if document posting works
uri:
url: http://localhost:8000/api/documents/post_document/
method: POST
body_format: form-multipart
body:
document:
content: FOO
filename: document.txt
mime_type: text/plain
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: post_document
failed_when: "'OK' not in post_document.content"
- name: verify uploaded document has been accepted
uri:
url: http://localhost:8000/api/logs/
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: logs
failed_when: "'Consuming document.txt' not in logs.content"
# assumes txt consumption finished by now, might have to sleep a bit
- name: verify uploaded document has been consumed
uri:
url: http://localhost:8000/api/logs/
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: logs
failed_when: "'document consumption finished' not in logs.content"
- name: verify uploaded document is avaiable
uri:
url: http://localhost:8000/api/documents/1/
headers:
Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
return_content: yes
register: document
failed_when: "'Not found.' in document.content or 'FOO' not in document.content"

View File

@ -0,0 +1,6 @@
---
- name: extract paperless-ng
unarchive:
src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
remote_src: yes
dest: "{{ tempdir.path }}"

View File

@ -0,0 +1,111 @@
---
- name: install dev dependencies
apt:
pkg:
- git
- npm
- gettext
- name: create output directories
file:
path: "{{ item }}"
state: directory
owner: "{{ paperlessng_system_user }}"
group: "{{ paperlessng_system_group }}"
mode: "750"
with_items:
- "{{ tempdir.path }}/paperless-ng"
- "{{ tempdir.path }}/paperless-ng/scripts"
- block:
- name: create temporary git directory
tempfile:
state: directory
path: "{{ paperlessng_directory }}"
register: gitdir
- name: pull paperless-ng
git:
repo: https://github.com/jonaswinkler/paperless-ng.git
dest: "{{ gitdir.path }}"
version: "{{ paperlessng_version }}"
refspec: "+refs/pull/*:refs/pull/*"
- name: compile frontend
command:
cmd: "{{ item }}"
args:
chdir: "{{ gitdir.path }}/src-ui"
failed_when: false
with_items:
- npm install -g @angular/cli
- npm install
- ./node_modules/.bin/ng build --prod
- name: copy application into place
copy:
src: "{{ gitdir.path }}/{{ item.src }}"
remote_src: yes
dest: "{{ tempdir.path }}/paperless-ng/{{ item.dest | default('') }}"
with_items:
- src: CONTRIBUTING.md
- src: LICENSE
- src: Pipfile
- src: Pipfile.lock
- src: README.md
- src: requirements.txt
- src: paperless.conf.example
dest: "paperless.conf"
- name: glob all scripts
find:
paths: ["{{ gitdir.path }}/scripts/"]
patterns:
- "*.service"
- "*.sh"
register: glob
- name: copy scripts
copy:
src: "{{ item.path }}"
remote_src: yes
dest: "{{ tempdir.path }}/paperless-ng/scripts/"
with_items:
- "{{ glob.files }}"
- name: copy sources
command:
cmd: "cp -r src/ {{ tempdir.path }}/paperless-ng/src"
args:
chdir: "{{ gitdir.path }}"
- name: create paperlessng venv
command:
cmd: "python3 -m virtualenv {{ gitdir.path }}/.venv/ -p /usr/bin/python3"
- name: install paperlessng requirements
command:
cmd: "{{ gitdir.path }}/.venv/bin/python3 -m pip install -r {{ gitdir.path }}/requirements.txt"
- name: compile messages
command: "{{ gitdir.path }}/.venv/bin/python3 manage.py compilemessages"
args:
chdir: "{{ tempdir.path }}/paperless-ng/src/"
- name: collect static files
command: "{{ gitdir.path }}/.venv/bin/python3 manage.py collectstatic --no-input"
args:
chdir: "{{ tempdir.path }}/paperless-ng/src/"
- name: remove pycache directories
shell: find . -name __pycache__ | xargs rm -r
args:
chdir: "{{ tempdir.path }}"
- name: remove temporary git directory
file:
path: "{{ gitdir.path }}"
state: absent
become: yes
become_user: "{{ paperlessng_system_user }}"

View File

@ -34,7 +34,13 @@
- build-essential
- python3-setuptools
- python3-wheel
- python3-virtualenv
# upstream virtualenv in Ubuntu 20.04 is broken
# https://github.com/pypa/virtualenv/issues/1873
- name: install python virtualenv
pip:
name: virtualenv
extra_args: --upgrade
- name: install ocr languages
apt:
@ -97,71 +103,141 @@
# GNUPG_HOME required due to paperless db.py
create_home: yes
- block:
- name: get latest release version
uri:
url: https://api.github.com/repos/jonaswinkler/paperless-ng/releases/latest
method: GET
register: latest_release
- name: parse latest release version
set_fact:
paperlessng_version: "{{ latest_release.json['tag_name'] }}"
when: paperlessng_version == "latest"
- block:
- name: sanitize version string
set_fact:
paperlessng_version: "{{ paperlessng_version | regex_replace('^ng-(\\d+\\.\\d+\\.\\d+)$', '\\1') }}"
- name: get tag data
uri:
url: https://api.github.com/repos/jonaswinkler/paperless-ng/tags
method: GET
register: tags
- name: get commit for target tag
set_fact:
paperlessng_commit: "{{ tags.json | json_query('[?name==`ng-' + paperlessng_version +'`] | [0].commit.sha') }}"
when: paperlessng_version | regex_search("^(ng-)?(\d+\.\d+\.\d+)$")
- block:
- name: check if version is branch
uri:
url: "https://api.github.com/repos/jonaswinkler/paperless-ng/branches/{{ paperlessng_version }}"
method: GET
status_code: [200, 404]
register: branch
- name: get commit for target branch
set_fact:
paperlessng_commit: "{{ branch.json | json_query('commit.sha') }}"
when: branch.status == 200
- block:
- name: check if version is commit-or-ref
uri:
url: "https://api.github.com/repos/jonaswinkler/paperless-ng/commits/{{ paperlessng_version }}"
method: GET
status_code: [200, 404, 422]
register: commit
- name: get commit for target commit-or-ref
set_fact:
paperlessng_commit: "{{ commit.json | json_query('sha') }}"
when: commit.status == 200
- name: fail
fail:
msg: "Can not determine commit from `paperlessng_version=={{ paperlessng_version }}`!"
when: commit.status != 200
when: branch.status == 404
when: not(paperlessng_version | regex_search("^(ng-)?(\d+\.\d+\.\d+)$"))
- name: check for paperless-ng installation
command:
cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html'
changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
cmd: "cat {{ paperlessng_directory }}/.installed_version"
changed_when: '"No such file or directory" in paperlessng_current_commit.stderr or paperlessng_current_commit.stdout != paperlessng_commit | string'
failed_when: false
ignore_errors: yes
register: paperlessng_current_version
register: paperlessng_current_commit
- name: register current state
set_fact:
fresh_installation: '{{ "No such file or directory" in paperlessng_current_version.stderr }}'
update_installation: '{{ "No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string }}'
reconfigure_only: '{{ paperlessng_current_version.stdout == paperlessng_version | string }}'
fresh_installation: '{{ "No such file or directory" in paperlessng_current_commit.stderr }}'
update_installation: '{{ "No such file or directory" not in paperlessng_current_commit.stderr and paperlessng_current_commit.stdout != paperlessng_commit | string }}'
reconfigure_only: "{{ paperlessng_current_commit.stdout == paperlessng_commit | string }}"
- name: backup current paperless-ng installation
copy:
src: "{{ paperlessng_directory }}"
remote_src: yes
dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/"
- block:
- name: backup current paperless-ng installation
copy:
src: "{{ paperlessng_directory }}"
remote_src: yes
dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/"
- name: remove current paperless sources
file:
path: "{{ paperlessng_directory }}/{{ item }}"
state: absent
with_items:
- docker
- docs
- scripts
- src
- static
when: update_installation
- name: remove current paperless sources
file:
path: "{{ paperlessng_directory }}/{{ item }}"
state: absent
with_items:
- docker
- docs
- scripts
- src
- static
when: update_installation
- name: create temporary directory
tempfile:
state: directory
register: tempdir
when: not reconfigure_only
- name: extract paperless-ng
unarchive:
src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
remote_src: yes
dest: "{{ tempdir.path }}"
when: not reconfigure_only
- name: change owner and permissions of paperless-ng
command:
cmd: "{{ item }}"
warn: false
with_items:
- "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
- "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
- "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
when: not reconfigure_only
- name: move paperless-ng
command:
cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}"
when: not reconfigure_only
- name: remove temporary directory
file:
path: "{{ tempdir.path }}"
state: absent
- block:
- name: create paperless-ng directory and set permissions
file:
path: "{{ paperlessng_directory }}"
state: directory
owner: "{{ paperlessng_system_user }}"
group: "{{ paperlessng_system_group }}"
mode: "750"
- name: create temporary directory
become: yes
become_user: "{{ paperlessng_system_user }}"
tempfile:
state: directory
path: "{{ paperlessng_directory }}"
register: tempdir
- name: check if version is available as release archive
uri:
url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
method: GET
status_code: [200, 404]
register: release_archive
- name: install paperless-ng from source
include_tasks: install-source.yml
when: release_archive.status == 404
- name: install paperless-ng from release archive
include_tasks: install-release.yml
when: release_archive.status == 200
- name: change owner and permissions of paperless-ng
command:
cmd: "{{ item }}"
warn: false
with_items:
- "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
- "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
- "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
- name: move paperless-ng
command:
cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}"
- name: store commit hash of installed version
copy:
content: "{{ paperlessng_commit }}"
dest: "{{ paperlessng_directory }}/.installed_version"
owner: "{{ paperlessng_system_user }}"
group: "{{ paperlessng_system_group }}"
mode: "0440"
- name: remove temporary directory
file:
path: "{{ tempdir.path }}"
state: absent
when: not reconfigure_only
- name: create paperless-ng directories and set permissions
@ -172,7 +248,6 @@
group: "{{ paperlessng_system_group }}"
mode: "750"
with_items:
- "{{ paperlessng_directory }}"
- "{{ paperlessng_consumption_dir }}"
- "{{ paperlessng_data_dir }}"
- "{{ paperlessng_media_root }}"
@ -180,7 +255,7 @@
- name: rename initial config
command:
cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template"
cmd: "mv -f {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template"
removes: "{{ paperlessng_directory }}/paperless.conf"
- name: configure paperless-ng
@ -310,21 +385,20 @@
creates: "{{ paperlessng_virtualenv }}"
register: venv
- name: install paperlessng requirements
become: yes
become_user: "{{ paperlessng_system_user }}"
pip:
requirements: "{{ paperlessng_directory }}/requirements.txt"
executable: "{{ paperlessng_virtualenv }}/bin/pip3"
extra_args: --upgrade
when: not reconfigure_only
- name: migrate database schema
become: yes
become_user: "{{ paperlessng_system_user }}"
command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate"
register: database_schema
changed_when: '"No migrations to apply." not in database_schema.stdout'
- block:
- name: install paperlessng requirements
become: yes
become_user: "{{ paperlessng_system_user }}"
pip:
requirements: "{{ paperlessng_directory }}/requirements.txt"
executable: "{{ paperlessng_virtualenv }}/bin/pip3"
extra_args: --upgrade
- name: migrate database schema
become: yes
become_user: "{{ paperlessng_system_user }}"
command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate"
register: database_schema
changed_when: '"No migrations to apply." not in database_schema.stdout'
when: not reconfigure_only
- name: configure paperless superuser
@ -392,7 +466,7 @@
# https://www.freedesktop.org/software/systemd/man/systemd.exec.html
{ option: "User", value: "{{ paperlessng_system_user }}" },
{ option: "Group", value: "{{ paperlessng_system_group }}" },
{ option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", },
{ option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src" },
{ option: "ProtectSystem", value: "full" },
{ option: "NoNewPrivileges", value: "true" },
{ option: "PrivateUsers", value: "true" },

View File

@ -381,6 +381,7 @@ class PostDocumentView(APIView):
with tempfile.NamedTemporaryFile(prefix="paperless-upload-",
dir=settings.SCRATCH_DIR,
buffering=0,
delete=False) as f:
f.write(doc_data)
os.utime(f.name, times=(t, t))