diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml
index fd965e760..a27ce103c 100644
--- a/.github/workflows/ansible.yml
+++ b/.github/workflows/ansible.yml
@@ -7,13 +7,20 @@ jobs:
   # https://molecule.readthedocs.io/en/latest/ci.html#github-actions
   test:
     runs-on: ubuntu-latest
-    # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context
-    if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/'))
     steps:
       - name: Check out the codebase
         uses: actions/checkout@v2
+        if: github.event_name != 'pull_request'
         with:
           path: "${{ github.repository }}"
+      - name: Check out the codebase
+        uses: actions/checkout@v2
+        if: github.event_name == 'pull_request'
+        with:
+          # merge commit is not available from tree at this point in time
+          # https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit
+          ref: "${{ github.event.pull_request.head.sha }}"
+          path: "${{ github.repository }}"
       - name: Set up Python
         uses: actions/setup-python@v2
       - name: Set up Docker
diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml
index f5f9b17c2..1ae260bac 100644
--- a/ansible/molecule/default/converge.yml
+++ b/ansible/molecule/default/converge.yml
@@ -2,9 +2,9 @@
 - name: update previous release to newest release
   hosts: all
   tasks:
-    - name: set github ref as version when available
+    - name: set current github commit as version when available
       set_fact:
-        paperlessng_version: "{{ lookup('env', 'GITHUB_REF') | default('latest', True) }}"
+        paperlessng_version: "{{ lookup('env', 'GITHUB_SHA') | default('master', True) }}"
     - name: update to newest paperless-ng release
       include_role:
         name: ansible
diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml
index f45747cb6..dba409549 100644
--- a/ansible/tasks/main.yml
+++ b/ansible/tasks/main.yml
@@ -114,6 +114,12 @@
         paperlessng_version: "{{ latest_release.json['tag_name'] }}"
   when: paperlessng_version == "latest"
 
+- name: check if version is ref
+  fail:
+    msg: "Specifying `paperlessng_version` as git ref may not work as expected!"
+  ignore_errors: True  # Output failure (as warning), but don't consider play failed
+  when: paperlessng_version.startswith('refs/')
+
 - block:
     - name: sanitize version string
       set_fact: