Merge pull request #367 from C0nsultant/ansible-verifyupdates

Ansible - Verify that updates between versions work

.github/workflows/ansible.yml

@@ -26,10 +26,15 @@ jobs:
           docker --version
           molecule --version
           python --version
-      - name: Test with molecule
+      - name: Test fresh installation with molecule
         run: |
           cd ansible
-          molecule test
+          molecule test -s fresh
+        working-directory: "${{ github.repository }}"
+      - name: Test release update with molecule
+        run: |
+          cd ansible
+          molecule test -s update
         working-directory: "${{ github.repository }}"
   # # https://galaxy.ansible.com/docs/contributing/importing.html
   # release:

ansible/molecule/default/converge.yml

@@ -1,7 +0,0 @@
----
-- name: Converge
-  hosts: all
-  tasks:
-    - name: "Include ansible"
-      include_role:
-        name: "ansible"

ansible/molecule/default/verify.yml

@@ -1,14 +0,0 @@
----
-# This is an example playbook to execute Ansible tests.
-
-- name: Verify
-  hosts: all
-  gather_facts: false
-  tasks:
-  - name: check if webserver is up
-    uri:
-      url: http://localhost:8000
-      status_code: [200, 302]
-      return_content: yes
-    register: landingpage
-    failed_when: "'Sign in</button>' not in landingpage.content"

ansible/molecule/fresh/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: fresh installation
+  hosts: all
+  tasks:
+    - name: install paperless-ng with default parameters
+      include_role:
+        name: ansible

ansible/molecule/fresh/verify.yml

@@ -0,0 +1,60 @@
+---
+- name: Verify
+  hosts: all
+  gather_facts: false
+
+  vars_files:
+    - ../../defaults/main.yml
+
+  tasks:
+    - name: check if webserver is up
+      uri:
+        url: http://localhost:8000
+        status_code: [200, 302]
+        return_content: yes
+      register: landingpage
+      failed_when: "'Sign in</button>' not in landingpage.content"
+
+    - name: check if document posting works
+      uri:
+        url: http://localhost:8000/api/documents/post_document/
+        method: POST
+        body_format: form-multipart
+        body:
+          document:
+            content: FOO
+            filename: document.txt
+            mime_type: text/plain
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: post_document
+      failed_when: "'OK' not in post_document.content"
+
+    - name: verify uploaded document has been accepted
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'Consuming document.txt' not in logs.content"
+
+    # assumes txt consumption finished by now, might have to sleep a bit
+    - name: verify uploaded document has been consumed
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'document consumption finished' not in logs.content"
+
+    - name: verify uploaded document is avaiable
+      uri:
+        url: http://localhost:8000/api/documents/1/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: document
+      failed_when: "'Not found.' in document.content or 'FOO' not in document.content"

ansible/molecule/update/converge.yml

@@ -0,0 +1,11 @@
+---
+- name: update previous release to newest release
+  hosts: all
+  tasks:
+    - name: set current version as installation target
+      set_fact:
+        paperlessng_version: 0.9.14
+
+    - name: update to newest paperless-ng release
+      include_role:
+        name: ansible

ansible/molecule/update/molecule.yml

@@ -0,0 +1,35 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: ubuntu_focal
+    image: jrei/systemd-ubuntu:20.04
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+      - /run/lock
+    override_command: False
+  # ubuntu 18.04 bionic works except that
+  #   the default redis configuration expects IPv6 which is not enabled in docker by default
+  #   the default Python environment is configured for ASCII instead of UTF-8
+  # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies
+  - name: debian_buster
+    image: jrei/systemd-debian:10
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+      - /run/lock
+    override_command: False
+  # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies
+provisioner:
+  name: ansible
+verifier:
+  name: ansible

ansible/molecule/update/prepare.yml

@@ -0,0 +1,10 @@
+- name: install previous release
+  hosts: all
+  tasks:
+    - name: set previous version as installation target
+      set_fact:
+        paperlessng_version: 0.9.13
+
+    - name: install previous paperless-ng release
+      include_role:
+        name: ansible

ansible/molecule/update/verify.yml

@@ -0,0 +1,60 @@
+---
+- name: Verify
+  hosts: all
+  gather_facts: false
+
+  vars_files:
+    - ../../defaults/main.yml
+
+  tasks:
+    - name: check if webserver is up
+      uri:
+        url: http://localhost:8000
+        status_code: [200, 302]
+        return_content: yes
+      register: landingpage
+      failed_when: "'Sign in</button>' not in landingpage.content"
+
+    - name: check if document posting works
+      uri:
+        url: http://localhost:8000/api/documents/post_document/
+        method: POST
+        body_format: form-multipart
+        body:
+          document:
+            content: FOO
+            filename: document.txt
+            mime_type: text/plain
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: post_document
+      failed_when: "'OK' not in post_document.content"
+
+    - name: verify uploaded document has been accepted
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'Consuming document.txt' not in logs.content"
+
+    # assumes txt consumption finished by now, might have to sleep a bit
+    - name: verify uploaded document has been consumed
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'document consumption finished' not in logs.content"
+
+    - name: verify uploaded document is avaiable
+      uri:
+        url: http://localhost:8000/api/documents/1/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: document
+      failed_when: "'Not found.' in document.content or 'FOO' not in document.content"

ansible/tasks/main.yml

@@ -105,25 +105,43 @@
   ignore_errors: yes
   register: paperlessng_current_version
 
+- name: register current state
+  set_fact:
+    fresh_installation: '{{ "No such file or directory" in paperlessng_current_version.stderr }}'
+    update_installation: '{{ "No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string }}'
+    reconfigure_only: '{{ paperlessng_current_version.stdout == paperlessng_version | string }}'
+
 - name: backup current paperless-ng installation
   copy:
     src: "{{ paperlessng_directory }}"
     remote_src: yes
     dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/"
-  when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string'
+  when: update_installation
+
+- name: remove current paperless sources
+  file:
+    path: "{{ paperlessng_directory }}/{{ item }}"
+    state: absent
+  with_items:
+    - docker
+    - docs
+    - scripts
+    - src
+    - static
+  when: update_installation
 
 - name: create temporary directory
   tempfile:
     state: directory
   register: tempdir
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  when: not reconfigure_only
 
 - name: extract paperless-ng
   unarchive:
     src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
     remote_src: yes
     dest: "{{ tempdir.path }}"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  when: not reconfigure_only
 
 - name: change owner and permissions of paperless-ng
   command:
@@ -133,18 +151,18 @@
     - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
     - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
     - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  when: not reconfigure_only
 
 - name: move paperless-ng
   command:
-    cmd: "cp -a {{ tempdir.path }}/paperless-ng/ {{ paperlessng_directory }}"
+    cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  when: not reconfigure_only
 
 - name: remove temporary directory
   file:
     path: "{{ tempdir.path }}"
     state: absent
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  when: not reconfigure_only
 
 - name: create paperless-ng directories and set permissions
   file:
@@ -281,7 +299,7 @@
     dest: /etc/paperless.conf
     owner: root
     group: root
-    mode: '0644'
+    mode: "0644"
   register: configuration
 
 - name: create paperlessng venv
@@ -299,15 +317,15 @@
     requirements: "{{ paperlessng_directory }}/requirements.txt"
     executable: "{{ paperlessng_virtualenv }}/bin/pip3"
     extra_args: --upgrade
-  when: paperlessng_current_version.stdout != paperlessng_version | string
+  when: not reconfigure_only
 
 - name: migrate database schema
   become: yes
   become_user: "{{ paperlessng_system_user }}"
   command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate"
-  when: paperlessng_current_version.stdout != paperlessng_version | string
   register: database_schema
   changed_when: '"No migrations to apply." not in database_schema.stdout'
+  when: not reconfigure_only
 
 - name: configure paperless superuser
   become: yes
@@ -336,7 +354,7 @@
       else:
           User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}')
           print('changed')
-  command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c \"{{ creation_script }}\""
+  command: '{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c "{{ creation_script }}"'
   register: superuser
   changed_when: superuser.stdout == 'changed'
   no_log: yes
@@ -349,7 +367,7 @@
     owner: "{{ paperlessng_system_user }}"
     group: "{{ paperlessng_system_group }}"
     mode: g-w,o-rwx
-  when: venv.changed or paperlessng_current_version.stdout != paperlessng_version | string
+  when: venv.changed or not reconfigure_only
 
 - name: configure ghostscript for PDF
   lineinfile:
@@ -372,34 +390,13 @@
       ]
     - [
         # https://www.freedesktop.org/software/systemd/man/systemd.exec.html
-        {
+        { option: "User", value: "{{ paperlessng_system_user }}" },
-          option: "User",
+        { option: "Group", value: "{{ paperlessng_system_group }}" },
-          value: "{{ paperlessng_system_user }}",
+        { option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", },
-        },
+        { option: "ProtectSystem", value: "full" },
-        {
+        { option: "NoNewPrivileges", value: "true" },
-          option: "Group",
+        { option: "PrivateUsers", value: "true" },
-          value: "{{ paperlessng_system_group }}",
+        { option: "PrivateDevices", value: "true" },
-        },
-        {
-          option: "WorkingDirectory",
-          value: "{{ paperlessng_directory }}/src",
-        },
-        {
-          option: "ProtectSystem",
-          value: "full",
-        },
-        {
-          option: "NoNewPrivileges",
-          value: "true",
-        },
-        {
-          option: "PrivateUsers",
-          value: "true",
-        },
-        {
-          option: "PrivateDevices",
-          value: "true",
-        }
       ]
 
 - name: configure paperless-consumer service

docs/administration.rst

@@ -53,6 +53,9 @@ Restoring
 Updating Paperless
 ##################
 
+Docker Route
+============
+
 If a new release of paperless-ng is available, upgrading depends on how you
 installed paperless-ng in the first place. The releases are available at the
 `release page <https://github.com/jonaswinkler/paperless-ng/releases>`_.
@@ -89,8 +92,8 @@ Running `docker-compose up` will also apply any new database migrations.
 If you see everything working, press CTRL+C once to gracefully stop paperless.
 Then you can start paperless-ng with ``-d`` to have it run in the background.
 
-Updating paperless without docker
+Bare Metal Route
-=================================
+================
 
 After grabbing the new release and unpacking the contents, do the following:
 
@@ -123,6 +126,38 @@ After grabbing the new release and unpacking the contents, do the following:
     This might not actually do anything. Not every new paperless version comes with new
     database migrations.
 
+Ansible Route
+=============
+
+Most of the update process is automated when using the ansible role.
+
+1.  Backup your defined role variables file outside the paperless source-tree:
+
+    .. code:: shell-session
+
+        $ cp ansible/vars.yml ~/vars.yml.old
+
+2.  Pull the release tag you want to update to:
+
+    .. code:: shell-session
+
+        $ git fetch --all
+        $ git checkout ng-0.9.14
+
+3.  Update the role variable definitions ``ansible/vars.yml`` (where appropriate).
+
+4.  Run the ansible playbook you created created during :ref:`installation <setup-ansible>` again:
+
+    .. note::
+
+        When ansible detects that an update run is in progress, it backs up the entire ``paperlessng_directory`` to ``paperlessng_directory-TIMESTAMP``.
+        Updates can be rolled back by simply moving the timestamped folder back to the original location.
+        If the update succeeds and you want to continue using the new release, please don't forget to delete the backup folder.
+
+    .. code:: shell-session
+
+        $ ansible-playbook playbook.yml
+
 
 Management utilities
 ####################