Merge pull request #367 from C0nsultant/ansible-verifyupdates

Ansible - Verify that updates between versions work

.github/workflows/ansible.yml

@@ -26,10 +26,15 @@ jobs:
           docker --version
           molecule --version
           python --version
-      - name: Test with molecule
+      - name: Test fresh installation with molecule
         run: |
           cd ansible
-          molecule test
+          molecule test -s fresh
+        working-directory: "${{ github.repository }}"
+      - name: Test release update with molecule
+        run: |
+          cd ansible
+          molecule test -s update
         working-directory: "${{ github.repository }}"
   # # https://galaxy.ansible.com/docs/contributing/importing.html
   # release:

ansible/molecule/default/converge.yml

@@ -1,7 +0,0 @@
----
-- name: Converge
-  hosts: all
-  tasks:
-    - name: "Include ansible"
-      include_role:
-        name: "ansible"

ansible/molecule/default/verify.yml

@@ -1,14 +0,0 @@
----
-# This is an example playbook to execute Ansible tests.
-
-- name: Verify
-  hosts: all
-  gather_facts: false
-  tasks:
-  - name: check if webserver is up
-    uri:
-      url: http://localhost:8000
-      status_code: [200, 302]
-      return_content: yes
-    register: landingpage
-    failed_when: "'Sign in</button>' not in landingpage.content"

ansible/molecule/fresh/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: fresh installation
+  hosts: all
+  tasks:
+    - name: install paperless-ng with default parameters
+      include_role:
+        name: ansible

ansible/molecule/fresh/verify.yml

@@ -0,0 +1,60 @@
+---
+- name: Verify
+  hosts: all
+  gather_facts: false
+
+  vars_files:
+    - ../../defaults/main.yml
+
+  tasks:
+    - name: check if webserver is up
+      uri:
+        url: http://localhost:8000
+        status_code: [200, 302]
+        return_content: yes
+      register: landingpage
+      failed_when: "'Sign in</button>' not in landingpage.content"
+
+    - name: check if document posting works
+      uri:
+        url: http://localhost:8000/api/documents/post_document/
+        method: POST
+        body_format: form-multipart
+        body:
+          document:
+            content: FOO
+            filename: document.txt
+            mime_type: text/plain
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: post_document
+      failed_when: "'OK' not in post_document.content"
+
+    - name: verify uploaded document has been accepted
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'Consuming document.txt' not in logs.content"
+
+    # assumes txt consumption finished by now, might have to sleep a bit
+    - name: verify uploaded document has been consumed
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'document consumption finished' not in logs.content"
+
+    - name: verify uploaded document is avaiable
+      uri:
+        url: http://localhost:8000/api/documents/1/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: document
+      failed_when: "'Not found.' in document.content or 'FOO' not in document.content"

ansible/molecule/update/converge.yml

@@ -0,0 +1,11 @@
+---
+- name: update previous release to newest release
+  hosts: all
+  tasks:
+    - name: set current version as installation target
+      set_fact:
+        paperlessng_version: 0.9.14
+
+    - name: update to newest paperless-ng release
+      include_role:
+        name: ansible

ansible/molecule/update/molecule.yml

@@ -0,0 +1,35 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: ubuntu_focal
+    image: jrei/systemd-ubuntu:20.04
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+      - /run/lock
+    override_command: False
+  # ubuntu 18.04 bionic works except that
+  #   the default redis configuration expects IPv6 which is not enabled in docker by default
+  #   the default Python environment is configured for ASCII instead of UTF-8
+  # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies
+  - name: debian_buster
+    image: jrei/systemd-debian:10
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+      - /run/lock
+    override_command: False
+  # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies
+provisioner:
+  name: ansible
+verifier:
+  name: ansible

ansible/molecule/update/prepare.yml

@@ -0,0 +1,10 @@
+- name: install previous release
+  hosts: all
+  tasks:
+    - name: set previous version as installation target
+      set_fact:
+        paperlessng_version: 0.9.13
+
+    - name: install previous paperless-ng release
+      include_role:
+        name: ansible

ansible/molecule/update/verify.yml

@@ -0,0 +1,60 @@
+---
+- name: Verify
+  hosts: all
+  gather_facts: false
+
+  vars_files:
+    - ../../defaults/main.yml
+
+  tasks:
+    - name: check if webserver is up
+      uri:
+        url: http://localhost:8000
+        status_code: [200, 302]
+        return_content: yes
+      register: landingpage
+      failed_when: "'Sign in</button>' not in landingpage.content"
+
+    - name: check if document posting works
+      uri:
+        url: http://localhost:8000/api/documents/post_document/
+        method: POST
+        body_format: form-multipart
+        body:
+          document:
+            content: FOO
+            filename: document.txt
+            mime_type: text/plain
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: post_document
+      failed_when: "'OK' not in post_document.content"
+
+    - name: verify uploaded document has been accepted
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'Consuming document.txt' not in logs.content"
+
+    # assumes txt consumption finished by now, might have to sleep a bit
+    - name: verify uploaded document has been consumed
+      uri:
+        url: http://localhost:8000/api/logs/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: logs
+      failed_when: "'document consumption finished' not in logs.content"
+
+    - name: verify uploaded document is avaiable
+      uri:
+        url: http://localhost:8000/api/documents/1/
+        headers:
+          Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}'
+        return_content: yes
+      register: document
+      failed_when: "'Not found.' in document.content or 'FOO' not in document.content"

ansible/tasks/main.yml

@@ -105,46 +105,64 @@
   ignore_errors: yes
   register: paperlessng_current_version
 
+- name: register current state
+  set_fact:
+    fresh_installation: '{{ "No such file or directory" in paperlessng_current_version.stderr }}'
+    update_installation: '{{ "No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string }}'
+    reconfigure_only: '{{ paperlessng_current_version.stdout == paperlessng_version | string }}'
+
 - name: backup current paperless-ng installation
   copy:
     src: "{{ paperlessng_directory }}"
     remote_src: yes
     dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/"
-  when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string'
+  when: update_installation
+
+- name: remove current paperless sources
+  file:
+    path: "{{ paperlessng_directory }}/{{ item }}"
+    state: absent
+  with_items:
+    - docker
+    - docs
+    - scripts
+    - src
+    - static
+  when: update_installation
 
 - name: create temporary directory
   tempfile:
     state: directory
   register: tempdir
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  when: not reconfigure_only
 
 - name: extract paperless-ng
   unarchive:
     src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz"
     remote_src: yes
     dest: "{{ tempdir.path }}"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  when: not reconfigure_only
 
 - name: change owner and permissions of paperless-ng
   command:
     cmd: "{{ item }}"
     warn: false
   with_items:
-  - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
-  - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
-  - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+    - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}"
+    - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;"
+    - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;"
+  when: not reconfigure_only
 
 - name: move paperless-ng
   command:
-    cmd: "cp -a {{ tempdir.path }}/paperless-ng/ {{ paperlessng_directory }}"
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+    cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}"
+  when: not reconfigure_only
 
 - name: remove temporary directory
   file:
     path: "{{ tempdir.path }}"
     state: absent
-  when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string'
+  when: not reconfigure_only
 
 - name: create paperless-ng directories and set permissions
   file:
@@ -281,7 +299,7 @@
     dest: /etc/paperless.conf
     owner: root
     group: root
-    mode: '0644'
+    mode: "0644"
   register: configuration
 
 - name: create paperlessng venv
@@ -299,15 +317,15 @@
     requirements: "{{ paperlessng_directory }}/requirements.txt"
     executable: "{{ paperlessng_virtualenv }}/bin/pip3"
     extra_args: --upgrade
-  when: paperlessng_current_version.stdout != paperlessng_version | string
+  when: not reconfigure_only
 
 - name: migrate database schema
   become: yes
   become_user: "{{ paperlessng_system_user }}"
   command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate"
-  when: paperlessng_current_version.stdout != paperlessng_version | string
   register: database_schema
   changed_when: '"No migrations to apply." not in database_schema.stdout'
+  when: not reconfigure_only
 
 - name: configure paperless superuser
   become: yes
@@ -336,7 +354,7 @@
       else:
           User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}')
           print('changed')
-  command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c \"{{ creation_script }}\""
+  command: '{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c "{{ creation_script }}"'
   register: superuser
   changed_when: superuser.stdout == 'changed'
   no_log: yes
@@ -349,7 +367,7 @@
     owner: "{{ paperlessng_system_user }}"
     group: "{{ paperlessng_system_group }}"
     mode: g-w,o-rwx
-  when: venv.changed or paperlessng_current_version.stdout != paperlessng_version | string
+  when: venv.changed or not reconfigure_only
 
 - name: configure ghostscript for PDF
   lineinfile:
@@ -372,34 +390,13 @@
       ]
     - [
         # https://www.freedesktop.org/software/systemd/man/systemd.exec.html
-        {
-          option: "User",
-          value: "{{ paperlessng_system_user }}",
-        },
-        {
-          option: "Group",
-          value: "{{ paperlessng_system_group }}",
-        },
-        {
-          option: "WorkingDirectory",
-          value: "{{ paperlessng_directory }}/src",
-        },
-        {
-          option: "ProtectSystem",
-          value: "full",
-        },
-        {
-          option: "NoNewPrivileges",
-          value: "true",
-        },
-        {
-          option: "PrivateUsers",
-          value: "true",
-        },
-        {
-          option: "PrivateDevices",
-          value: "true",
-        }
+        { option: "User", value: "{{ paperlessng_system_user }}" },
+        { option: "Group", value: "{{ paperlessng_system_group }}" },
+        { option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", },
+        { option: "ProtectSystem", value: "full" },
+        { option: "NoNewPrivileges", value: "true" },
+        { option: "PrivateUsers", value: "true" },
+        { option: "PrivateDevices", value: "true" },
       ]
 
 - name: configure paperless-consumer service

docs/administration.rst

@@ -53,6 +53,9 @@ Restoring
 Updating Paperless
 ##################
 
+Docker Route
+============
+
 If a new release of paperless-ng is available, upgrading depends on how you
 installed paperless-ng in the first place. The releases are available at the
 `release page <https://github.com/jonaswinkler/paperless-ng/releases>`_.
@@ -72,10 +75,10 @@ A.  If you pull the image from the docker hub, all you need to do is:
 
         $ docker-compose pull
         $ docker-compose up
-    
+
     The docker-compose files refer to the ``latest`` version, which is always the latest
     stable release.
-        
+
 B.  If you built the image yourself, do the following:
 
     .. code:: shell-session
@@ -89,13 +92,13 @@ Running `docker-compose up` will also apply any new database migrations.
 If you see everything working, press CTRL+C once to gracefully stop paperless.
 Then you can start paperless-ng with ``-d`` to have it run in the background.
 
-Updating paperless without docker
-=================================
+Bare Metal Route
+================
 
 After grabbing the new release and unpacking the contents, do the following:
 
 1.  Update dependencies. New paperless version may require additional
-    dependencies. The dependencies required are listed in the section about 
+    dependencies. The dependencies required are listed in the section about
     :ref:`bare metal installations <setup-bare_metal>`.
 
 2.  Update python requirements. If you use Pipenv, this is done with the following steps.
@@ -112,18 +115,50 @@ After grabbing the new release and unpacking the contents, do the following:
 
     You can also use the included ``requirements.txt`` file instead and create the virtual
     environment yourself. This file includes exactly the same dependencies.
-    
+
 3.  Migrate the database.
 
     .. code:: shell-session
 
         $ cd src
         $ pipenv run python3 manage.py migrate
-    
+
     This might not actually do anything. Not every new paperless version comes with new
     database migrations.
-    
-        
+
+Ansible Route
+=============
+
+Most of the update process is automated when using the ansible role.
+
+1.  Backup your defined role variables file outside the paperless source-tree:
+
+    .. code:: shell-session
+
+        $ cp ansible/vars.yml ~/vars.yml.old
+
+2.  Pull the release tag you want to update to:
+
+    .. code:: shell-session
+
+        $ git fetch --all
+        $ git checkout ng-0.9.14
+
+3.  Update the role variable definitions ``ansible/vars.yml`` (where appropriate).
+
+4.  Run the ansible playbook you created created during :ref:`installation <setup-ansible>` again:
+
+    .. note::
+
+        When ansible detects that an update run is in progress, it backs up the entire ``paperlessng_directory`` to ``paperlessng_directory-TIMESTAMP``.
+        Updates can be rolled back by simply moving the timestamped folder back to the original location.
+        If the update succeeds and you want to continue using the new release, please don't forget to delete the backup folder.
+
+    .. code:: shell-session
+
+        $ ansible-playbook playbook.yml
+
+
 Management utilities
 ####################
 
@@ -361,7 +396,7 @@ Documents can be stored in Paperless using GnuPG encryption.
     Furthermore, the entire text content of the documents is stored plain in the
     database, even if your documents are encrypted. Filenames are not encrypted as
     well.
-    
+
     Also, the web server provides transparent access to your encrypted documents.
 
     Consider running paperless on an encrypted filesystem instead, which will then
@@ -386,4 +421,4 @@ Basic usage to disable encryption of your document store:
     decrypt_documents [--passphrase SECR3TP4SSPHRA$E]
 
 
-.. _Pipenv: https://pipenv.pypa.io/en/latest/
+.. _Pipenv: https://pipenv.pypa.io/en/latest/