diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..715474a85 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,62 @@ +name: docker + +on: + push: + branches: + - master + - dev + tags: + - 'ng-*' + +jobs: + # build and push image to docker hub. + buildx: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Prepare + id: prepare + env: + DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} + run: | + DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + fi + if [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=${GITHUB_REF#refs/heads/} + fi + TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION == "master" ]]; then + TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" + fi + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${VERSION} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${TAGS} --file ./Dockerfile . + - name: install buildx + id: buildx + uses: crazy-max/ghaction-docker-buildx@v3.1.0 + - name: Docker Login + if: success() && github.event_name != 'pull_request' + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + - name: Docker Buildx (push) + if: success() && github.event_name != 'pull_request' + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Check Manifest + if: always() && github.event_name != 'pull_request' + run: | + docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + - name: Clear + if: always() && github.event_name != 'pull_request' + run: | + rm -f ${HOME}/.docker/config.json diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 000000000..00835d240 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,33 @@ +name: documentation + +# always run documentation workflow to ensure the documentation can still be built +on: [push, pull_request] + +# possibly add a publish job for automatic read-the-docs publishing +jobs: + documentation: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "3.6.8" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip3.6.8} + - name: dependencies + # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? + run: | + pip install 'sphinx~=3.3' sphinx_rtd_theme + - name: make + run: | + cd docs/ + make html \ No newline at end of file diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 000000000..4cb36d3a4 --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,19 @@ +name: frontend + +# always run frontend workflow to see if it builds and tests run fine +on: [push, pull_request] + +jobs: + frontend: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: build + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + ng build --prod \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..9766c4b5b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +name: backend-tests + +# always run tests workflow +on: [push, pull_request] + +jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + python: # pyenv-action requires the exact version + - 3.6.8 + - 3.7.9 + - 3.8.6 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "${{ matrix.python }}" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python }} + - name: Tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + cd src/ + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f4873cf3c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,153 +0,0 @@ -language: python - -dist: focal -os: linux - -stages: - - name: test - - name: build_docker - if: branch =~ /^(master|dev|ng-.*)$/ - - name: publish_manifest - if: branch =~ /^(master|dev|ng-.*)$/ - -jobs: - include: - - name: "Paperless on Python 3.6" - stage: test - python: "3.6" - - - name: "Paperless on Python 3.7" - stage: test - python: "3.7" - - - name: "Paperless on Python 3.8" - stage: test - python: "3.8" - - - name: "Documentation" - stage: test - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - stage: test - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true - - - stage: build_docker - name: amd64 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - - stage: build_docker - name: arm64v8 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64-graviton2 - virt: vm - group: edge - - stage: build_docker - name: arm32v7 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - - stage: publish_manifest - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} - - | - if [ "${TRAVIS_BRANCH}" = "master" ]; then - echo "Master branch detected" - DOCKER_TAG="latest" - else - DOCKER_TAG=${TRAVIS_TAG} - fi - - | - if [ "${DOCKER_TAG}" != "" ]; then - echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} - else - echo "Not a tag and not on master, so not pushing tag/master specific manifest" - fi - - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - -install: - - pip install --upgrade pipenv - - pipenv install --system --dev - -script: - - cd src/ - - pipenv run pytest --cov - - pipenv run pycodestyle - -after_success: - - pipenv run coveralls