From eca1289ce2f34f09254341d60801eb3d94ff9bcb Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 9 Jan 2021 10:49:44 +0100 Subject: [PATCH 01/11] Build release archive when version is not avail Default ansible installation version to "latest" (pulls the latest published release archive). --- ansible/defaults/main.yml | 2 +- ansible/tasks/install-release.yml | 6 ++ ansible/tasks/install-source.yml | 133 +++++++++++++++++++++++++++ ansible/tasks/main.yml | 148 +++++++++++++++++------------- 4 files changed, 222 insertions(+), 67 deletions(-) create mode 100644 ansible/tasks/install-release.yml create mode 100644 ansible/tasks/install-source.yml diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 83047307d..aaeffa507 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.14 +paperlessng_version: latest # 'latest', release number, or github branch/tag/commit/ref # Required services paperlessng_redis_host: localhost diff --git a/ansible/tasks/install-release.yml b/ansible/tasks/install-release.yml new file mode 100644 index 000000000..c2dfb0b9f --- /dev/null +++ b/ansible/tasks/install-release.yml @@ -0,0 +1,6 @@ +--- +- name: extract paperless-ng + unarchive: + src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + remote_src: yes + dest: "{{ tempdir.path }}" diff --git a/ansible/tasks/install-source.yml b/ansible/tasks/install-source.yml new file mode 100644 index 000000000..823445dd1 --- /dev/null +++ b/ansible/tasks/install-source.yml @@ -0,0 +1,133 @@ +--- +# https://github.com/jonaswinkler/paperless-ng/blob/dev/.github/workflows/ci.yml +- name: install dev dependencies + apt: + pkg: + - git + - npm + - libqpdf-dev + +- name: create temporary git directory + tempfile: + state: directory + register: gitdir + +- name: pull paperless-ng + git: + repo: https://github.com/jonaswinkler/paperless-ng.git + dest: "{{ gitdir.path }}" + version: "{{ paperlessng_version }}" + refspec: "+refs/pull/*:refs/pull/*" + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: compile frontend + command: + cmd: "{{ item }}" + args: + chdir: "{{ gitdir.path }}/src-ui" + failed_when: false + with_items: + - npm install -g @angular/cli + - npm install + - ./node_modules/.bin/ng build --prod + +- name: install pipenv + pip: + name: + - pipenv + - pybind11 # building pikepdf for <0.9.14 + extra_args: --upgrade + +- name: allow building with any Python 3 release + lineinfile: + path: "{{ gitdir.path }}/Pipfile" + regexp: '^python_version = ".+"$' + line: python_version = "3" + +# TODO run dev in separate virtualenv +- name: install Pipfile dependencies + command: + cmd: pipenv install --dev + args: + chdir: "{{ gitdir.path }}" + +- name: clean output directory + file: + path: "{{ gitdir.path }}/dist" + state: absent + +- name: create output directories + file: + path: "{{ item }}" + state: directory + with_items: + - "{{ gitdir.path }}/dist" + - "{{ gitdir.path }}/dist/paperless-ng" + - "{{ gitdir.path }}/dist/paperless-ng/scripts" + +- name: copy application into place + copy: + src: "{{ gitdir.path }}/{{ item.src }}" + remote_src: yes + dest: "{{ gitdir.path }}/dist/paperless-ng/{{ item.dest | default('') }}" + with_items: + - src: CONTRIBUTING.md + - src: LICENSE + - src: Pipfile + - src: Pipfile.lock + - src: README.md + - src: paperless.conf.example + dest: "paperless.conf" + +# TODO can be copied for >=0.9.14 +- name: generate requirements.txt + command: + cmd: pipenv lock --keep-outdated -r + args: + chdir: "{{ gitdir.path }}" + register: requirements + +- name: write requirements.txt + copy: + content: "{{ requirements.stdout }}" + dest: "{{ gitdir.path }}/dist/paperless-ng/requirements.txt" + +- name: glob all scripts + find: + paths: "{{ gitdir.path }}/scripts/" + patterns: + - "*.service" + - "*.sh" + register: glob + +- name: copy scripts + copy: + src: "{{ item.path }}" + remote_src: yes + dest: "{{ gitdir.path }}/dist/paperless-ng/scripts/" + with_items: + - "{{ glob.files }}" + +- name: copy sources + command: + cmd: "cp -r src/ dist/paperless-ng/src" + args: + chdir: "{{ gitdir.path }}" + +- name: package app + archive: + path: "{{ gitdir.path }}/dist/" + dest: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + format: xz + +- name: extract paperless-ng + unarchive: + src: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + remote_src: yes + dest: "{{ tempdir.path }}" + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: remove temporary git directory + file: + path: "{{ gitdir.path }}" + state: absent diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index a353a18ec..15f7f9ba1 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -34,7 +34,13 @@ - build-essential - python3-setuptools - python3-wheel - - python3-virtualenv + +# upstream virtualenv in Ubuntu 20.04 is broken +# https://github.com/pypa/virtualenv/issues/1873 +- name: install python virtualenv + pip: + name: virtualenv + extra_args: --upgrade - name: install ocr languages apt: @@ -97,6 +103,18 @@ # GNUPG_HOME required due to paperless db.py create_home: yes +- block: + - name: get latest release version + uri: + url: https://api.github.com/repos/jonaswinkler/paperless-ng/releases/latest + method: GET + register: latest_release + - name: parse latest release version + set_fact: + paperlessng_version: "{{ latest_release.json['tag_name'] | regex_replace('^ng-(.+)$', '\\1') }}" + when: paperlessng_version == "latest" + +# TODO store commit hash of installed version, use instead of version number - name: check for paperless-ng installation command: cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' @@ -109,59 +127,58 @@ set_fact: fresh_installation: '{{ "No such file or directory" in paperlessng_current_version.stderr }}' update_installation: '{{ "No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string }}' - reconfigure_only: '{{ paperlessng_current_version.stdout == paperlessng_version | string }}' + reconfigure_only: "{{ paperlessng_current_version.stdout == paperlessng_version | string }}" -- name: backup current paperless-ng installation - copy: - src: "{{ paperlessng_directory }}" - remote_src: yes - dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" +- block: + - name: backup current paperless-ng installation + copy: + src: "{{ paperlessng_directory }}" + remote_src: yes + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" + - name: remove current paperless sources + file: + path: "{{ paperlessng_directory }}/{{ item }}" + state: absent + with_items: + - docker + - docs + - scripts + - src + - static when: update_installation -- name: remove current paperless sources - file: - path: "{{ paperlessng_directory }}/{{ item }}" - state: absent - with_items: - - docker - - docs - - scripts - - src - - static - when: update_installation - -- name: create temporary directory - tempfile: - state: directory - register: tempdir - when: not reconfigure_only - -- name: extract paperless-ng - unarchive: - src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" - remote_src: yes - dest: "{{ tempdir.path }}" - when: not reconfigure_only - -- name: change owner and permissions of paperless-ng - command: - cmd: "{{ item }}" - warn: false - with_items: - - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}" - - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" - - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" - when: not reconfigure_only - -- name: move paperless-ng - command: - cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" - when: not reconfigure_only - -- name: remove temporary directory - file: - path: "{{ tempdir.path }}" - state: absent +- block: + - name: create temporary directory + tempfile: + state: directory + register: tempdir + - name: check if version is available as release archive + uri: + url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + method: GET + status_code: [200, 302, 404] + register: release_archive + - name: install paperless-ng from source + include_tasks: install-source.yml + when: release_archive.status == 404 + - name: install paperless-ng from release archive + include_tasks: install-release.yml + when: release_archive.status != 404 + - name: change owner and permissions of paperless-ng + command: + cmd: "{{ item }}" + warn: false + with_items: + - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}" + - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" + - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" + - name: move paperless-ng + command: + cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" + - name: remove temporary directory + file: + path: "{{ tempdir.path }}" + state: absent when: not reconfigure_only - name: create paperless-ng directories and set permissions @@ -310,21 +327,20 @@ creates: "{{ paperlessng_virtualenv }}" register: venv -- name: install paperlessng requirements - become: yes - become_user: "{{ paperlessng_system_user }}" - pip: - requirements: "{{ paperlessng_directory }}/requirements.txt" - executable: "{{ paperlessng_virtualenv }}/bin/pip3" - extra_args: --upgrade - when: not reconfigure_only - -- name: migrate database schema - become: yes - become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" - register: database_schema - changed_when: '"No migrations to apply." not in database_schema.stdout' +- block: + - name: install paperlessng requirements + become: yes + become_user: "{{ paperlessng_system_user }}" + pip: + requirements: "{{ paperlessng_directory }}/requirements.txt" + executable: "{{ paperlessng_virtualenv }}/bin/pip3" + extra_args: --upgrade + - name: migrate database schema + become: yes + become_user: "{{ paperlessng_system_user }}" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + register: database_schema + changed_when: '"No migrations to apply." not in database_schema.stdout' when: not reconfigure_only - name: configure paperless superuser From 8d624937744abf6ed3ce4d521178e1d62f1dfc64 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Fri, 22 Jan 2021 11:10:56 +0100 Subject: [PATCH 02/11] Adapt github action to build PR version --- .github/workflows/ansible.yml | 23 ++++++++++++++++++++++- ansible/molecule/fresh/converge.yml | 6 ++++++ ansible/molecule/update/converge.yml | 5 ++--- ansible/molecule/update/prepare.yml | 2 +- ansible/tasks/install-source.yml | 4 +--- 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 646c7ff81..60589f87d 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: # https://molecule.readthedocs.io/en/latest/ci.html#github-actions - test: + test-fresh: runs-on: ubuntu-latest # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/')) @@ -31,6 +31,27 @@ jobs: cd ansible molecule test -s fresh working-directory: "${{ github.repository }}" + test-update: + runs-on: ubuntu-latest + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/')) + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Set up Docker + uses: docker-practice/actions-setup-docker@master + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install molecule[ansible,docker] + ansible --version + docker --version + molecule --version + python --version - name: Test release update with molecule run: | cd ansible diff --git a/ansible/molecule/fresh/converge.yml b/ansible/molecule/fresh/converge.yml index 99e25677b..39d20050d 100644 --- a/ansible/molecule/fresh/converge.yml +++ b/ansible/molecule/fresh/converge.yml @@ -2,6 +2,12 @@ - name: fresh installation hosts: all tasks: + - name: set github ref as version when available + set_fact: + paperlessng_version: "{{ lookup('env', 'GITHUB_REF') | default('latest', True) }}" + - name: debug + debug: + var: paperlessng_version - name: install paperless-ng with default parameters include_role: name: ansible diff --git a/ansible/molecule/update/converge.yml b/ansible/molecule/update/converge.yml index b19a5981a..f5f9b17c2 100644 --- a/ansible/molecule/update/converge.yml +++ b/ansible/molecule/update/converge.yml @@ -2,10 +2,9 @@ - name: update previous release to newest release hosts: all tasks: - - name: set current version as installation target + - name: set github ref as version when available set_fact: - paperlessng_version: 0.9.14 - + paperlessng_version: "{{ lookup('env', 'GITHUB_REF') | default('latest', True) }}" - name: update to newest paperless-ng release include_role: name: ansible diff --git a/ansible/molecule/update/prepare.yml b/ansible/molecule/update/prepare.yml index 6f3734329..138ebdfce 100644 --- a/ansible/molecule/update/prepare.yml +++ b/ansible/molecule/update/prepare.yml @@ -3,7 +3,7 @@ tasks: - name: set previous version as installation target set_fact: - paperlessng_version: 0.9.13 + paperlessng_version: 1.0.0 - name: install previous paperless-ng release include_role: diff --git a/ansible/tasks/install-source.yml b/ansible/tasks/install-source.yml index 823445dd1..8f6dc0e03 100644 --- a/ansible/tasks/install-source.yml +++ b/ansible/tasks/install-source.yml @@ -18,7 +18,6 @@ dest: "{{ gitdir.path }}" version: "{{ paperlessng_version }}" refspec: "+refs/pull/*:refs/pull/*" - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: compile frontend command: @@ -31,6 +30,7 @@ - npm install - ./node_modules/.bin/ng build --prod +# TODO run dev in separate virtualenv - name: install pipenv pip: name: @@ -44,7 +44,6 @@ regexp: '^python_version = ".+"$' line: python_version = "3" -# TODO run dev in separate virtualenv - name: install Pipfile dependencies command: cmd: pipenv install --dev @@ -125,7 +124,6 @@ src: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_version }}.tar.xz" remote_src: yes dest: "{{ tempdir.path }}" - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary git directory file: From 6c3b1db4dd3d9086057bdd7da82bb3748daeba37 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Fri, 22 Jan 2021 14:21:35 +0100 Subject: [PATCH 03/11] Determine installed version by git commit hash --- .github/workflows/ansible.yml | 4 +- ansible/molecule/fresh/converge.yml | 3 -- ansible/tasks/install-source.yml | 4 +- ansible/tasks/main.yml | 73 ++++++++++++++++++++++++----- 4 files changed, 65 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 60589f87d..c11472361 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install molecule[ansible,docker] + python3 -m pip install molecule[ansible,docker] jmespath ansible --version docker --version molecule --version @@ -47,7 +47,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install molecule[ansible,docker] + python3 -m pip install molecule[ansible,docker] jmespath ansible --version docker --version molecule --version diff --git a/ansible/molecule/fresh/converge.yml b/ansible/molecule/fresh/converge.yml index 39d20050d..eec6e6444 100644 --- a/ansible/molecule/fresh/converge.yml +++ b/ansible/molecule/fresh/converge.yml @@ -5,9 +5,6 @@ - name: set github ref as version when available set_fact: paperlessng_version: "{{ lookup('env', 'GITHUB_REF') | default('latest', True) }}" - - name: debug - debug: - var: paperlessng_version - name: install paperless-ng with default parameters include_role: name: ansible diff --git a/ansible/tasks/install-source.yml b/ansible/tasks/install-source.yml index 8f6dc0e03..64b7dbeb2 100644 --- a/ansible/tasks/install-source.yml +++ b/ansible/tasks/install-source.yml @@ -116,12 +116,12 @@ - name: package app archive: path: "{{ gitdir.path }}/dist/" - dest: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + dest: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_commit }}.tar.xz" format: xz - name: extract paperless-ng unarchive: - src: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + src: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_commit }}.tar.xz" remote_src: yes dest: "{{ tempdir.path }}" diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 15f7f9ba1..5ad30dfcf 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -111,23 +111,65 @@ register: latest_release - name: parse latest release version set_fact: - paperlessng_version: "{{ latest_release.json['tag_name'] | regex_replace('^ng-(.+)$', '\\1') }}" + paperlessng_version: "{{ latest_release.json['tag_name'] }}" when: paperlessng_version == "latest" -# TODO store commit hash of installed version, use instead of version number +- block: + - name: sanitize version string + set_fact: + paperlessng_version: "{{ paperlessng_version | regex_replace('^ng-(\\d+\\.\\d+\\.\\d+)$', '\\1') }}" + - name: get tag data + uri: + url: https://api.github.com/repos/jonaswinkler/paperless-ng/tags + method: GET + register: tags + - name: get commit for target tag + set_fact: + paperlessng_commit: "{{ tags.json | json_query('[?name==`ng-' + paperlessng_version +'`] | [0].commit.sha') }}" + when: paperlessng_version | regex_search("^(ng-)?(\d+\.\d+\.\d+)$") + +- block: + - name: check if version is branch + uri: + url: "https://api.github.com/repos/jonaswinkler/paperless-ng/branches/{{ paperlessng_version }}" + method: GET + status_code: [200, 404] + register: branch + - name: get commit for target branch + set_fact: + paperlessng_commit: "{{ branch.json | json_query('commit.sha') }}" + when: branch.status == 200 + - block: + - name: check if version is commit-or-ref + uri: + url: "https://api.github.com/repos/jonaswinkler/paperless-ng/commits/{{ paperlessng_version }}" + method: GET + status_code: [200, 404, 422] + register: commit + - name: get commit for target commit-or-ref + set_fact: + paperlessng_commit: "{{ commit.json | json_query('sha') }}" + when: commit.status == 200 + - name: fail + fail: + msg: "Can not determine commit from `paperlessng_version=={{ paperlessng_version }}`!" + when: commit.status != 200 + when: branch.status == 404 + when: not(paperlessng_version | regex_search("^(ng-)?(\d+\.\d+\.\d+)$")) + - name: check for paperless-ng installation command: - cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' - changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + cmd: "cat {{ paperlessng_directory }}/.installed_version" + changed_when: '"No such file or directory" in paperlessng_current_commit.stderr or paperlessng_current_commit.stdout != paperlessng_commit | string' failed_when: false ignore_errors: yes - register: paperlessng_current_version + register: paperlessng_current_commit - name: register current state set_fact: - fresh_installation: '{{ "No such file or directory" in paperlessng_current_version.stderr }}' - update_installation: '{{ "No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string }}' - reconfigure_only: "{{ paperlessng_current_version.stdout == paperlessng_version | string }}" + fresh_installation: '{{ "No such file or directory" in paperlessng_current_commit.stderr }}' + update_installation: '{{ "No such file or directory" not in paperlessng_current_commit.stderr and paperlessng_current_commit.stdout != paperlessng_commit | string }}' + reconfigure_only: "{{ paperlessng_current_commit.stdout == paperlessng_commit | string }}" - block: - name: backup current paperless-ng installation @@ -156,14 +198,14 @@ uri: url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" method: GET - status_code: [200, 302, 404] + status_code: [200, 404] register: release_archive - name: install paperless-ng from source include_tasks: install-source.yml when: release_archive.status == 404 - name: install paperless-ng from release archive include_tasks: install-release.yml - when: release_archive.status != 404 + when: release_archive.status == 200 - name: change owner and permissions of paperless-ng command: cmd: "{{ item }}" @@ -175,6 +217,13 @@ - name: move paperless-ng command: cmd: "cp -a {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" + - name: store commit hash of installed version + copy: + content: "{{ paperlessng_commit }}" + dest: "{{ paperlessng_directory }}/.installed_version" + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: "0440" - name: remove temporary directory file: path: "{{ tempdir.path }}" @@ -197,7 +246,7 @@ - name: rename initial config command: - cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template" + cmd: "mv -f {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template" removes: "{{ paperlessng_directory }}/paperless.conf" - name: configure paperless-ng @@ -408,7 +457,7 @@ # https://www.freedesktop.org/software/systemd/man/systemd.exec.html { option: "User", value: "{{ paperlessng_system_user }}" }, { option: "Group", value: "{{ paperlessng_system_group }}" }, - { option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", }, + { option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src" }, { option: "ProtectSystem", value: "full" }, { option: "NoNewPrivileges", value: "true" }, { option: "PrivateUsers", value: "true" }, From 0d19957d4b96c8414c46d45e21517065fc0f875e Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 23 Jan 2021 22:08:36 +0100 Subject: [PATCH 04/11] Fully prepare release package --- ansible/tasks/install-source.yml | 63 +++++++++++++------------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/ansible/tasks/install-source.yml b/ansible/tasks/install-source.yml index 64b7dbeb2..04a2991b9 100644 --- a/ansible/tasks/install-source.yml +++ b/ansible/tasks/install-source.yml @@ -5,7 +5,7 @@ pkg: - git - npm - - libqpdf-dev + - gettext - name: create temporary git directory tempfile: @@ -30,26 +30,6 @@ - npm install - ./node_modules/.bin/ng build --prod -# TODO run dev in separate virtualenv -- name: install pipenv - pip: - name: - - pipenv - - pybind11 # building pikepdf for <0.9.14 - extra_args: --upgrade - -- name: allow building with any Python 3 release - lineinfile: - path: "{{ gitdir.path }}/Pipfile" - regexp: '^python_version = ".+"$' - line: python_version = "3" - -- name: install Pipfile dependencies - command: - cmd: pipenv install --dev - args: - chdir: "{{ gitdir.path }}" - - name: clean output directory file: path: "{{ gitdir.path }}/dist" @@ -75,28 +55,16 @@ - src: Pipfile - src: Pipfile.lock - src: README.md + - src: requirements.txt - src: paperless.conf.example dest: "paperless.conf" -# TODO can be copied for >=0.9.14 -- name: generate requirements.txt - command: - cmd: pipenv lock --keep-outdated -r - args: - chdir: "{{ gitdir.path }}" - register: requirements - -- name: write requirements.txt - copy: - content: "{{ requirements.stdout }}" - dest: "{{ gitdir.path }}/dist/paperless-ng/requirements.txt" - - name: glob all scripts find: - paths: "{{ gitdir.path }}/scripts/" + paths: ["{{ gitdir.path }}/scripts/"] patterns: - - "*.service" - - "*.sh" + - "*.service" + - "*.sh" register: glob - name: copy scripts @@ -113,6 +81,27 @@ args: chdir: "{{ gitdir.path }}" +- name: install paperlessng requirements + pip: + requirements: "{{ gitdir.path }}/requirements.txt" + virtualenv: "{{ gitdir.path }}/.venv/" + extra_args: --upgrade + +- name: compile messages + command: "{{ gitdir.path }}/.venv/bin/python3 manage.py compilemessages" + args: + chdir: "{{ gitdir.path }}/dist/paperless-ng/src/" + +- name: collect static files + command: "{{ gitdir.path }}/.venv/bin/python3 manage.py collectstatic --no-input" + args: + chdir: "{{ gitdir.path }}/dist/paperless-ng/src/" + +- name: remove pycache directories + shell: find . -name __pycache__ | xargs rm -r + args: + chdir: "{{ gitdir.path }}/dist/" + - name: package app archive: path: "{{ gitdir.path }}/dist/" From bfbdfe857f28d47dd99fda1e8e12689de142faf7 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 23 Jan 2021 22:09:56 +0100 Subject: [PATCH 05/11] Simplify molecule tests "Upgrade" path includes multiple paths anyway: - installing the latest official release package - builing the current PR from source - upgrading between the two versions --- .github/workflows/ansible.yml | 37 ++------ .../molecule/{update => default}/converge.yml | 0 .../molecule/{fresh => default}/molecule.yml | 0 .../molecule/{update => default}/prepare.yml | 2 +- ansible/molecule/default/verify.yml | 91 +++++++++++++++++++ ansible/molecule/fresh/converge.yml | 10 -- ansible/molecule/fresh/verify.yml | 60 ------------ ansible/molecule/update/molecule.yml | 35 ------- ansible/molecule/update/verify.yml | 60 ------------ 9 files changed, 100 insertions(+), 195 deletions(-) rename ansible/molecule/{update => default}/converge.yml (100%) rename ansible/molecule/{fresh => default}/molecule.yml (100%) rename ansible/molecule/{update => default}/prepare.yml (85%) create mode 100644 ansible/molecule/default/verify.yml delete mode 100644 ansible/molecule/fresh/converge.yml delete mode 100644 ansible/molecule/fresh/verify.yml delete mode 100644 ansible/molecule/update/molecule.yml delete mode 100644 ansible/molecule/update/verify.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index c11472361..fd965e760 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: # https://molecule.readthedocs.io/en/latest/ci.html#github-actions - test-fresh: + test: runs-on: ubuntu-latest # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/')) @@ -26,36 +26,15 @@ jobs: docker --version molecule --version python --version - - name: Test fresh installation with molecule + - name: Test installation/build/upgrade with molecule run: | cd ansible - molecule test -s fresh - working-directory: "${{ github.repository }}" - test-update: - runs-on: ubuntu-latest - # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context - if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/')) - steps: - - name: Check out the codebase - uses: actions/checkout@v2 - with: - path: "${{ github.repository }}" - - name: Set up Python - uses: actions/setup-python@v2 - - name: Set up Docker - uses: docker-practice/actions-setup-docker@master - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - python3 -m pip install molecule[ansible,docker] jmespath - ansible --version - docker --version - molecule --version - python --version - - name: Test release update with molecule - run: | - cd ansible - molecule test -s update + molecule create + molecule verify + molecule converge + molecule idempotence + molecule verify + molecule destroy working-directory: "${{ github.repository }}" # # https://galaxy.ansible.com/docs/contributing/importing.html # release: diff --git a/ansible/molecule/update/converge.yml b/ansible/molecule/default/converge.yml similarity index 100% rename from ansible/molecule/update/converge.yml rename to ansible/molecule/default/converge.yml diff --git a/ansible/molecule/fresh/molecule.yml b/ansible/molecule/default/molecule.yml similarity index 100% rename from ansible/molecule/fresh/molecule.yml rename to ansible/molecule/default/molecule.yml diff --git a/ansible/molecule/update/prepare.yml b/ansible/molecule/default/prepare.yml similarity index 85% rename from ansible/molecule/update/prepare.yml rename to ansible/molecule/default/prepare.yml index 138ebdfce..e175eff5b 100644 --- a/ansible/molecule/update/prepare.yml +++ b/ansible/molecule/default/prepare.yml @@ -3,7 +3,7 @@ tasks: - name: set previous version as installation target set_fact: - paperlessng_version: 1.0.0 + paperlessng_version: latest - name: install previous paperless-ng release include_role: diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml new file mode 100644 index 000000000..01dc43192 --- /dev/null +++ b/ansible/molecule/default/verify.yml @@ -0,0 +1,91 @@ +--- +- name: Verify + hosts: all + gather_facts: false + + vars_files: + - ../../defaults/main.yml + + tasks: + - name: check if webserver is up + uri: + url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}" + status_code: [200, 302] + return_content: yes + register: landingpage + failed_when: "'Sign in' not in landingpage.content" + + - name: generate random name and content + set_fact: + content: "{{ lookup('password', '/dev/null length=64 chars=ascii_letters') }}" + filename: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters') }}" + + - name: check if document posting works + uri: + url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/post_document/" + method: POST + body_format: form-multipart + body: + document: + content: "{{ content }}" + filename: "{{ filename }}.txt" + mime_type: text/plain + headers: + Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' + return_content: yes + register: post_document + failed_when: "'OK' not in post_document.content" + + - name: verify uploaded document has been accepted + uri: + url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/" + headers: + Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' + return_content: yes + register: logs + failed_when: "('Consuming ' + filename + '.txt') not in logs.content" + + # assumes txt consumption finished by now, might have to sleep a bit + - name: verify uploaded document has been consumed + uri: + url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/" + headers: + Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' + return_content: yes + register: logs + failed_when: "filename + ' consumption finished' not in logs.content" + + - name: get documents + uri: + url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/" + headers: + Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' + return_content: yes + register: documents + + - name: set document index + set_fact: + index: "{{ documents.json['results'][0]['id'] }}" + + - name: verify uploaded document is avaiable + uri: + url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/{{ index }}/" + headers: + Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' + return_content: yes + register: document + failed_when: "'Not found.' in document.content or content not in document.json['content']" + + - name: check if deleting uploaded document works + uri: + url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/documents/bulk_edit/" + method: POST + body_format: json + body: + documents: ["{{ index }}"] + method: delete + parameters: {} + headers: + Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' + register: delete_document + failed_when: "'OK' not in delete_document.json['result']" diff --git a/ansible/molecule/fresh/converge.yml b/ansible/molecule/fresh/converge.yml deleted file mode 100644 index eec6e6444..000000000 --- a/ansible/molecule/fresh/converge.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: fresh installation - hosts: all - tasks: - - name: set github ref as version when available - set_fact: - paperlessng_version: "{{ lookup('env', 'GITHUB_REF') | default('latest', True) }}" - - name: install paperless-ng with default parameters - include_role: - name: ansible diff --git a/ansible/molecule/fresh/verify.yml b/ansible/molecule/fresh/verify.yml deleted file mode 100644 index c353783ab..000000000 --- a/ansible/molecule/fresh/verify.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - - vars_files: - - ../../defaults/main.yml - - tasks: - - name: check if webserver is up - uri: - url: http://localhost:8000 - status_code: [200, 302] - return_content: yes - register: landingpage - failed_when: "'Sign in' not in landingpage.content" - - - name: check if document posting works - uri: - url: http://localhost:8000/api/documents/post_document/ - method: POST - body_format: form-multipart - body: - document: - content: FOO - filename: document.txt - mime_type: text/plain - headers: - Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' - return_content: yes - register: post_document - failed_when: "'OK' not in post_document.content" - - - name: verify uploaded document has been accepted - uri: - url: http://localhost:8000/api/logs/ - headers: - Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' - return_content: yes - register: logs - failed_when: "'Consuming document.txt' not in logs.content" - - # assumes txt consumption finished by now, might have to sleep a bit - - name: verify uploaded document has been consumed - uri: - url: http://localhost:8000/api/logs/ - headers: - Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' - return_content: yes - register: logs - failed_when: "'document consumption finished' not in logs.content" - - - name: verify uploaded document is avaiable - uri: - url: http://localhost:8000/api/documents/1/ - headers: - Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' - return_content: yes - register: document - failed_when: "'Not found.' in document.content or 'FOO' not in document.content" diff --git a/ansible/molecule/update/molecule.yml b/ansible/molecule/update/molecule.yml deleted file mode 100644 index 27f37ba63..000000000 --- a/ansible/molecule/update/molecule.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -platforms: - - name: ubuntu_focal - image: jrei/systemd-ubuntu:20.04 - privileged: true - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - tmpfs: - - /tmp - - /run - - /run/lock - override_command: False - # ubuntu 18.04 bionic works except that - # the default redis configuration expects IPv6 which is not enabled in docker by default - # the default Python environment is configured for ASCII instead of UTF-8 - # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies - - name: debian_buster - image: jrei/systemd-debian:10 - privileged: true - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - tmpfs: - - /tmp - - /run - - /run/lock - override_command: False - # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies -provisioner: - name: ansible -verifier: - name: ansible diff --git a/ansible/molecule/update/verify.yml b/ansible/molecule/update/verify.yml deleted file mode 100644 index c353783ab..000000000 --- a/ansible/molecule/update/verify.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - - vars_files: - - ../../defaults/main.yml - - tasks: - - name: check if webserver is up - uri: - url: http://localhost:8000 - status_code: [200, 302] - return_content: yes - register: landingpage - failed_when: "'Sign in' not in landingpage.content" - - - name: check if document posting works - uri: - url: http://localhost:8000/api/documents/post_document/ - method: POST - body_format: form-multipart - body: - document: - content: FOO - filename: document.txt - mime_type: text/plain - headers: - Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' - return_content: yes - register: post_document - failed_when: "'OK' not in post_document.content" - - - name: verify uploaded document has been accepted - uri: - url: http://localhost:8000/api/logs/ - headers: - Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' - return_content: yes - register: logs - failed_when: "'Consuming document.txt' not in logs.content" - - # assumes txt consumption finished by now, might have to sleep a bit - - name: verify uploaded document has been consumed - uri: - url: http://localhost:8000/api/logs/ - headers: - Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' - return_content: yes - register: logs - failed_when: "'document consumption finished' not in logs.content" - - - name: verify uploaded document is avaiable - uri: - url: http://localhost:8000/api/documents/1/ - headers: - Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' - return_content: yes - register: document - failed_when: "'Not found.' in document.content or 'FOO' not in document.content" From be2013975c170c9b8c2f36a5a4407dcfcaea1917 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sun, 24 Jan 2021 09:59:54 +0100 Subject: [PATCH 06/11] Simplify building from source Do not package app, instead copy directly into expected directory --- ansible/tasks/install-source.yml | 34 ++++++++------------------------ 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/ansible/tasks/install-source.yml b/ansible/tasks/install-source.yml index 04a2991b9..64fbc525b 100644 --- a/ansible/tasks/install-source.yml +++ b/ansible/tasks/install-source.yml @@ -30,25 +30,19 @@ - npm install - ./node_modules/.bin/ng build --prod -- name: clean output directory - file: - path: "{{ gitdir.path }}/dist" - state: absent - - name: create output directories file: path: "{{ item }}" state: directory with_items: - - "{{ gitdir.path }}/dist" - - "{{ gitdir.path }}/dist/paperless-ng" - - "{{ gitdir.path }}/dist/paperless-ng/scripts" + - "{{ tempdir.path }}/paperless-ng" + - "{{ tempdir.path }}/paperless-ng/scripts" - name: copy application into place copy: src: "{{ gitdir.path }}/{{ item.src }}" remote_src: yes - dest: "{{ gitdir.path }}/dist/paperless-ng/{{ item.dest | default('') }}" + dest: "{{ tempdir.path }}/paperless-ng/{{ item.dest | default('') }}" with_items: - src: CONTRIBUTING.md - src: LICENSE @@ -71,13 +65,13 @@ copy: src: "{{ item.path }}" remote_src: yes - dest: "{{ gitdir.path }}/dist/paperless-ng/scripts/" + dest: "{{ tempdir.path }}/paperless-ng/scripts/" with_items: - "{{ glob.files }}" - name: copy sources command: - cmd: "cp -r src/ dist/paperless-ng/src" + cmd: "cp -r src/ {{ tempdir.path }}/paperless-ng/src" args: chdir: "{{ gitdir.path }}" @@ -90,29 +84,17 @@ - name: compile messages command: "{{ gitdir.path }}/.venv/bin/python3 manage.py compilemessages" args: - chdir: "{{ gitdir.path }}/dist/paperless-ng/src/" + chdir: "{{ tempdir.path }}/paperless-ng/src/" - name: collect static files command: "{{ gitdir.path }}/.venv/bin/python3 manage.py collectstatic --no-input" args: - chdir: "{{ gitdir.path }}/dist/paperless-ng/src/" + chdir: "{{ tempdir.path }}/paperless-ng/src/" - name: remove pycache directories shell: find . -name __pycache__ | xargs rm -r args: - chdir: "{{ gitdir.path }}/dist/" - -- name: package app - archive: - path: "{{ gitdir.path }}/dist/" - dest: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_commit }}.tar.xz" - format: xz - -- name: extract paperless-ng - unarchive: - src: "{{ gitdir.path }}/paperless-ng-{{ paperlessng_commit }}.tar.xz" - remote_src: yes - dest: "{{ tempdir.path }}" + chdir: "{{ tempdir.path }}" - name: remove temporary git directory file: From 29ce2515eea976294e838537215316c2f6d71925 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sun, 24 Jan 2021 10:05:42 +0100 Subject: [PATCH 07/11] Build source package in paperlessng_directory Avoids permission problems in /tmp --- ansible/molecule/default/verify.yml | 5 +- ansible/tasks/install-source.yml | 165 +++++++++++++++------------- ansible/tasks/main.yml | 11 +- 3 files changed, 101 insertions(+), 80 deletions(-) diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 01dc43192..1b3a436ca 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -45,7 +45,10 @@ register: logs failed_when: "('Consuming ' + filename + '.txt') not in logs.content" - # assumes txt consumption finished by now, might have to sleep a bit + - name: sleep 5 seconds + pause: + seconds: 5 + - name: verify uploaded document has been consumed uri: url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/" diff --git a/ansible/tasks/install-source.yml b/ansible/tasks/install-source.yml index 64fbc525b..ab8fbfef7 100644 --- a/ansible/tasks/install-source.yml +++ b/ansible/tasks/install-source.yml @@ -1,5 +1,4 @@ --- -# https://github.com/jonaswinkler/paperless-ng/blob/dev/.github/workflows/ci.yml - name: install dev dependencies apt: pkg: @@ -7,96 +6,106 @@ - npm - gettext -- name: create temporary git directory - tempfile: - state: directory - register: gitdir - -- name: pull paperless-ng - git: - repo: https://github.com/jonaswinkler/paperless-ng.git - dest: "{{ gitdir.path }}" - version: "{{ paperlessng_version }}" - refspec: "+refs/pull/*:refs/pull/*" - -- name: compile frontend - command: - cmd: "{{ item }}" - args: - chdir: "{{ gitdir.path }}/src-ui" - failed_when: false - with_items: - - npm install -g @angular/cli - - npm install - - ./node_modules/.bin/ng build --prod - - name: create output directories file: path: "{{ item }}" state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: "750" with_items: - "{{ tempdir.path }}/paperless-ng" - "{{ tempdir.path }}/paperless-ng/scripts" -- name: copy application into place - copy: - src: "{{ gitdir.path }}/{{ item.src }}" - remote_src: yes - dest: "{{ tempdir.path }}/paperless-ng/{{ item.dest | default('') }}" - with_items: - - src: CONTRIBUTING.md - - src: LICENSE - - src: Pipfile - - src: Pipfile.lock - - src: README.md - - src: requirements.txt - - src: paperless.conf.example - dest: "paperless.conf" +- block: + - name: create temporary git directory + tempfile: + state: directory + path: "{{ paperlessng_directory }}" + register: gitdir -- name: glob all scripts - find: - paths: ["{{ gitdir.path }}/scripts/"] - patterns: - - "*.service" - - "*.sh" - register: glob + - name: pull paperless-ng + git: + repo: https://github.com/jonaswinkler/paperless-ng.git + dest: "{{ gitdir.path }}" + version: "{{ paperlessng_version }}" + refspec: "+refs/pull/*:refs/pull/*" -- name: copy scripts - copy: - src: "{{ item.path }}" - remote_src: yes - dest: "{{ tempdir.path }}/paperless-ng/scripts/" - with_items: - - "{{ glob.files }}" + - name: compile frontend + command: + cmd: "{{ item }}" + args: + chdir: "{{ gitdir.path }}/src-ui" + failed_when: false + with_items: + - npm install -g @angular/cli + - npm install + - ./node_modules/.bin/ng build --prod -- name: copy sources - command: - cmd: "cp -r src/ {{ tempdir.path }}/paperless-ng/src" - args: - chdir: "{{ gitdir.path }}" + - name: copy application into place + copy: + src: "{{ gitdir.path }}/{{ item.src }}" + remote_src: yes + dest: "{{ tempdir.path }}/paperless-ng/{{ item.dest | default('') }}" + with_items: + - src: CONTRIBUTING.md + - src: LICENSE + - src: Pipfile + - src: Pipfile.lock + - src: README.md + - src: requirements.txt + - src: paperless.conf.example + dest: "paperless.conf" -- name: install paperlessng requirements - pip: - requirements: "{{ gitdir.path }}/requirements.txt" - virtualenv: "{{ gitdir.path }}/.venv/" - extra_args: --upgrade + - name: glob all scripts + find: + paths: ["{{ gitdir.path }}/scripts/"] + patterns: + - "*.service" + - "*.sh" + register: glob -- name: compile messages - command: "{{ gitdir.path }}/.venv/bin/python3 manage.py compilemessages" - args: - chdir: "{{ tempdir.path }}/paperless-ng/src/" + - name: copy scripts + copy: + src: "{{ item.path }}" + remote_src: yes + dest: "{{ tempdir.path }}/paperless-ng/scripts/" + with_items: + - "{{ glob.files }}" -- name: collect static files - command: "{{ gitdir.path }}/.venv/bin/python3 manage.py collectstatic --no-input" - args: - chdir: "{{ tempdir.path }}/paperless-ng/src/" + - name: copy sources + command: + cmd: "cp -r src/ {{ tempdir.path }}/paperless-ng/src" + args: + chdir: "{{ gitdir.path }}" -- name: remove pycache directories - shell: find . -name __pycache__ | xargs rm -r - args: - chdir: "{{ tempdir.path }}" + - name: create paperlessng venv + command: + cmd: "python3 -m virtualenv {{ gitdir.path }}/.venv/ -p /usr/bin/python3" -- name: remove temporary git directory - file: - path: "{{ gitdir.path }}" - state: absent + - name: install paperlessng requirements + command: + cmd: "{{ gitdir.path }}/.venv/bin/python3 -m pip install -r {{ gitdir.path }}/requirements.txt" + + - name: compile messages + command: "{{ gitdir.path }}/.venv/bin/python3 manage.py compilemessages" + args: + chdir: "{{ tempdir.path }}/paperless-ng/src/" + + - name: collect static files + command: "{{ gitdir.path }}/.venv/bin/python3 manage.py collectstatic --no-input" + args: + chdir: "{{ tempdir.path }}/paperless-ng/src/" + + - name: remove pycache directories + shell: find . -name __pycache__ | xargs rm -r + args: + chdir: "{{ tempdir.path }}" + + - name: remove temporary git directory + file: + path: "{{ gitdir.path }}" + state: absent + + become: yes + become_user: "{{ paperlessng_system_user }}" diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 5ad30dfcf..f45747cb6 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -190,9 +190,19 @@ when: update_installation - block: + - name: create paperless-ng directory and set permissions + file: + path: "{{ paperlessng_directory }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: "750" - name: create temporary directory + become: yes + become_user: "{{ paperlessng_system_user }}" tempfile: state: directory + path: "{{ paperlessng_directory }}" register: tempdir - name: check if version is available as release archive uri: @@ -238,7 +248,6 @@ group: "{{ paperlessng_system_group }}" mode: "750" with_items: - - "{{ paperlessng_directory }}" - "{{ paperlessng_consumption_dir }}" - "{{ paperlessng_data_dir }}" - "{{ paperlessng_media_root }}" From 21c501de28f09c582b78dbce45b825c57feef84e Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 27 Jan 2021 07:17:46 +0100 Subject: [PATCH 08/11] force flush on temp file during consumption file.write() does not guarantee that a file handle contains anything without calling an accompanying file.flush() For typical files that are larger than the OS-file buffer, this is no problem For small files (e.g. 64 characters in a .TXT), this race condition leads to `inode/x-empty` because the file inode is created, but no content is written to it --- ansible/molecule/default/verify.yml | 6 +----- src/documents/views.py | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 1b3a436ca..4d7e30f4d 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -29,9 +29,9 @@ document: content: "{{ content }}" filename: "{{ filename }}.txt" - mime_type: text/plain headers: Authorization: 'Basic {{ (paperlessng_superuser_name + ":" + paperlessng_superuser_password) | b64encode }}' + Content-Type: text/plain return_content: yes register: post_document failed_when: "'OK' not in post_document.content" @@ -45,10 +45,6 @@ register: logs failed_when: "('Consuming ' + filename + '.txt') not in logs.content" - - name: sleep 5 seconds - pause: - seconds: 5 - - name: verify uploaded document has been consumed uri: url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/" diff --git a/src/documents/views.py b/src/documents/views.py index b99bf11c7..7aadba36d 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -383,6 +383,7 @@ class PostDocumentView(APIView): dir=settings.SCRATCH_DIR, delete=False) as f: f.write(doc_data) + f.flush() os.utime(f.name, times=(t, t)) async_task("documents.tasks.consume_file", From 063bfc245cca390fd6745b95fac2ae6875c6e836 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 30 Jan 2021 12:20:55 +0100 Subject: [PATCH 09/11] DEBUG - force newline in .txt --- ansible/molecule/default/verify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 4d7e30f4d..fde2e8905 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -17,7 +17,7 @@ - name: generate random name and content set_fact: - content: "{{ lookup('password', '/dev/null length=64 chars=ascii_letters') }}" + content: "{{ lookup('password', '/dev/null length=64 chars=ascii_letters') }}\n" filename: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters') }}" - name: check if document posting works From d9e06958dcd1dc1888f16efc614dce7feb57f667 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 30 Jan 2021 12:50:05 +0100 Subject: [PATCH 10/11] DEBUG - force flush 2nd try --- ansible/molecule/default/verify.yml | 2 +- src/documents/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index fde2e8905..4d7e30f4d 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -17,7 +17,7 @@ - name: generate random name and content set_fact: - content: "{{ lookup('password', '/dev/null length=64 chars=ascii_letters') }}\n" + content: "{{ lookup('password', '/dev/null length=64 chars=ascii_letters') }}" filename: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters') }}" - name: check if document posting works diff --git a/src/documents/views.py b/src/documents/views.py index 7aadba36d..808ecb925 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -381,9 +381,9 @@ class PostDocumentView(APIView): with tempfile.NamedTemporaryFile(prefix="paperless-upload-", dir=settings.SCRATCH_DIR, + buffering=0, delete=False) as f: f.write(doc_data) - f.flush() os.utime(f.name, times=(t, t)) async_task("documents.tasks.consume_file", From 269673ce05c15ceb05cdda6907c207878ec14d4c Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Sat, 30 Jan 2021 13:58:54 +0100 Subject: [PATCH 11/11] DEBUG - use very long txt file --- ansible/molecule/default/verify.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 4d7e30f4d..185840783 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -17,7 +17,7 @@ - name: generate random name and content set_fact: - content: "{{ lookup('password', '/dev/null length=64 chars=ascii_letters') }}" + content: "{{ lookup('password', '/dev/null length=65536 chars=ascii_letters') }}" filename: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters') }}" - name: check if document posting works @@ -45,6 +45,10 @@ register: logs failed_when: "('Consuming ' + filename + '.txt') not in logs.content" + - name: sleep till consumption finished + pause: + seconds: 10 + - name: verify uploaded document has been consumed uri: url: "http://{{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}/api/logs/"