From 548ffa851b37c9f0e28e4ae542b9fb72779128c5 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Mon, 30 Mar 2020 20:29:08 +0200 Subject: [PATCH] Github actions enhancements (#319) --- .circleci/config.yml | 4 +- .github/script/deploy | 8 + .github/script/docker | 36 ++ .github/workflows/build.yml | 345 ++++++++++++++++++ .github/workflows/deploy.yml | 176 --------- script/compile | 56 +-- {.circleci/script => script}/install-clojure | 2 +- .../script => script}/install-leiningen | 7 +- script/test | 3 + 9 files changed, 413 insertions(+), 224 deletions(-) create mode 100755 .github/script/deploy create mode 100755 .github/script/docker create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/deploy.yml rename {.circleci/script => script}/install-clojure (95%) rename {.circleci/script => script}/install-leiningen (50%) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9c105ad..191af3d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -218,11 +218,11 @@ jobs: - run: name: Install Clojure command: | - .circleci/script/install-clojure /usr/local + script/install-clojure /usr/local - run: name: Install Leiningen command: | - .circleci/script/install-leiningen + script/install-leiningen - run: name: Download GraalVM command: | diff --git a/.github/script/deploy b/.github/script/deploy new file mode 100755 index 00000000..31c8a147 --- /dev/null +++ b/.github/script/deploy @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [ -z "$GITHUB_HEAD_REF" ] && [ "${GITHUB_REF##*/}" = "master" ] +then + lein deploy clojars +fi + +exit 0; diff --git a/.github/script/docker b/.github/script/docker new file mode 100755 index 00000000..a0121d94 --- /dev/null +++ b/.github/script/docker @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -eo pipefail + +image_name="borkdude/babashka" +image_tag=$(cat resources/BABASHKA_VERSION) +latest_tag="latest" + +if [[ $image_tag =~ SNAPSHOT$ ]] +then + echo "This is a snapshot version" + snapshot="true" +else + echo "This is a non-snapshot version" + snapshot="false" +fi + +if [ -z "$GITHUB_HEAD_REF" ] && [ "${GITHUB_REF##*/}" = "master" ] +then + echo "Building Docker image $image_name:$image_tag" + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin + docker build -t "$image_name" . + docker tag "$image_name:$latest_tag" "$image_name:$image_tag" + # we only update latest when it's not a SNAPSHOT version + if [ "false" = "$snapshot" ]; then + echo "Pushing image $image_name:$latest_tag" + docker push "$image_name:$latest_tag" + fi + # we update the version tag, even if it's a SNAPSHOT version + echo "Pushing image $image_name:$image_tag" + docker push "$image_name:$image_tag" +else + echo "Not publishing Docker image" +fi + +exit 0; diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..ae444cf3 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,345 @@ +name: build + +on: [push + , pull_request + ] + +jobs: + + scratch: + runs-on: ubuntu-18.04 + steps: + - name: Git checkout + uses: actions/checkout@v1 + with: + fetch-depth: 1 + submodules: 'true' + + - name: Scratch + run: | + echo "Scratch" + + jvm: + # ubuntu 18.04 comes with lein + java8 installed + runs-on: ubuntu-18.04 + steps: + - name: Git checkout + uses: actions/checkout@v1 + with: + fetch-depth: 1 + submodules: 'true' + + - name: Cache deps + uses: actions/cache@v1 + id: cache-deps + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Fetch deps + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + lein deps + + - name: Run tests + run: | + script/test + + - name: Test libraries + run: | + sudo script/install-clojure + script/run_lib_tests + + - name: Build uberjar + run: | + lein with-profiles +reflection do run + lein do clean, uberjar + + - name: Babashka version + id: babashka-version + run: | + BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION) + echo "##[set-output name=version;]${BABASHKA_VERSION}" + + - name: Reflection artifact + run: | + cp reflection.json babashka-${{ steps.babashka-version.outputs.version }}-reflection.json + + - uses: actions/upload-artifact@v1 + with: + name: jar + path: target/babashka-${{ steps.babashka-version.outputs.version }}-standalone.jar + + - uses: actions/upload-artifact@v1 + with: + name: reflection.json + path: babashka-${{ steps.babashka-version.outputs.version }}-reflection.json + + linux: + needs: [jvm] + runs-on: ubuntu-18.04 + steps: + - name: Git checkout + uses: actions/checkout@v1 + with: + fetch-depth: 1 + submodules: 'true' + + - uses: actions/download-artifact@v1 + with: + name: jar + path: . + + - uses: actions/download-artifact@v1 + with: + name: reflection.json + path: . + + - name: Cache deps + uses: actions/cache@v1 + id: cache-deps + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Cache GraalVM + uses: actions/cache@v1 + id: cache-graalvm + with: + path: ~/graalvm-ce-java8-19.3.1 + key: ${{ runner.os }}-graalvm-19.3.1 + restore-keys: | + ${{ runner.os }}-graalvm-19.3.1 + + - name: Download GraalVM + run: | + cd ~ + if ! [ -d graalvm-ce-java8-19.3.1 ]; then + curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-19.3.1/graalvm-ce-java8-linux-amd64-19.3.1.tar.gz + tar xzf graalvm-ce-java8-linux-amd64-19.3.1.tar.gz + fi + + - name: Babashka version + id: babashka-version + run: | + BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION) + echo "##[set-output name=version;]${BABASHKA_VERSION}" + + - name: Build Linux native image + run: | + export BABASHKA_JAR=babashka-${{ steps.babashka-version.outputs.version }}-standalone.jar + export BABASHKA_XMX="-J-Xmx6g" + export GRAALVM_HOME="$HOME/graalvm-ce-java8-19.3.1" + cp babashka-${{ steps.babashka-version.outputs.version }}-reflection.json reflection.json + script/compile + + - name: Test binary + run: | + BABASHKA_TEST_ENV=native script/test + + - name: Install clojure + run: | + sudo script/install-clojure /usr/local + + - name: Test libraries + run: | + BABASHKA_TEST_ENV=native script/run_lib_tests + + - uses: actions/upload-artifact@v1 + with: + path: bb + name: babashka-${{ steps.babashka-version.outputs.version }}-linux-amd64.zip + + linux-static: + needs: [jvm] + runs-on: ubuntu-16.04 + steps: + - name: Git checkout + uses: actions/checkout@v1 + with: + fetch-depth: 1 + submodules: 'true' + + - uses: actions/download-artifact@v1 + with: + name: jar + path: . + + - uses: actions/download-artifact@v1 + with: + name: reflection.json + path: . + + - name: Cache deps + uses: actions/cache@v1 + id: cache-deps + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Cache GraalVM + uses: actions/cache@v1 + id: cache-graalvm + with: + path: ~/graalvm-ce-java8-19.3.1 + key: ${{ runner.os }}-graalvm-19.3.1 + restore-keys: | + ${{ runner.os }}-graalvm-19.3.1 + + - name: Download GraalVM + run: | + cd ~ + if ! [ -d graalvm-ce-java8-19.3.1 ]; then + curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-19.3.1/graalvm-ce-java8-linux-amd64-19.3.1.tar.gz + tar xzf graalvm-ce-java8-linux-amd64-19.3.1.tar.gz + fi + + - name: Babashka version + id: babashka-version + run: | + BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION) + echo "##[set-output name=version;]${BABASHKA_VERSION}" + + - name: Build Linux native image + run: | + export BABASHKA_JAR=babashka-${{ steps.babashka-version.outputs.version }}-standalone.jar + export BABASHKA_XMX="-J-Xmx6g" + export GRAALVM_HOME="$HOME/graalvm-ce-java8-19.3.1" + export BABASHKA_STATIC=true + cp babashka-${{ steps.babashka-version.outputs.version }}-reflection.json reflection.json + script/compile + + - name: Test binary + run: | + ./bb '(+ 1 2 3)' + BABASHKA_TEST_ENV=native script/test + + - name: Install clojure + run: | + sudo script/install-clojure + + - name: Test libraries + run: | + BABASHKA_TEST_ENV=native script/run_lib_tests + + - uses: actions/upload-artifact@v1 + with: + path: bb + name: babashka-${{ steps.babashka-version.outputs.version }}-linux-static-amd64.zip + + mac: + needs: [jvm] + runs-on: macOS-latest + steps: + - name: Git checkout + uses: actions/checkout@v1 + with: + fetch-depth: 1 + submodules: 'true' + + - uses: actions/download-artifact@v1 + with: + name: jar + path: . + + - uses: actions/download-artifact@v1 + with: + name: reflection.json + path: . + + - name: Cache GraalVM + uses: actions/cache@v1 + id: cache-graalvm + with: + path: ~/graalvm-ce-java8-19.3.1 + key: ${{ runner.os }}-graalvm-19.3.1 + restore-keys: | + ${{ runner.os }}-graalvm-19.3.1 + + - name: Download GraalVM + run: | + cd ~ + if ! [ -d graalvm-ce-java8-19.3.1 ]; then + curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-19.3.1/graalvm-ce-java8-darwin-amd64-19.3.1.tar.gz + tar xzf graalvm-ce-java8-darwin-amd64-19.3.1.tar.gz + fi + + - name: Babashka version + id: babashka-version + run: | + BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION) + echo "##[set-output name=version;]${BABASHKA_VERSION}" + + - name: Build macOS native image + run: | + export BABASHKA_JAR=babashka-${{ steps.babashka-version.outputs.version }}-standalone.jar + export BABASHKA_XMX="-J-Xmx6g" + export GRAALVM_HOME="$HOME/graalvm-ce-java8-19.3.1/Contents/Home" + cp babashka-${{ steps.babashka-version.outputs.version }}-reflection.json reflection.json + script/compile + + - name: Test binary + run: | + sudo script/install-leiningen + BABASHKA_TEST_ENV=native script/test + + - name: Test libraries + run: | + sudo script/install-clojure + BABASHKA_TEST_ENV=native script/run_lib_tests + + - uses: actions/upload-artifact@v1 + with: + path: bb + name: babashka-${{ steps.babashka-version.outputs.version }}-macos-amd64.zip + + deploy: + needs: [jvm, linux, linux-static, mac] + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + runs-on: ubuntu-18.04 + steps: + - name: Git checkout + uses: actions/checkout@v1 + with: + fetch-depth: 1 + submodules: 'true' + + - name: Cache deps + uses: actions/cache@v1 + id: cache-deps + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Deploy + env: + CLOJARS_USER: "${{ secrets.CLOJARS_USER }}" + CLOJARS_PASS: "${{ secrets.CLOJARS_PASS }}" + run: | + .github/script/deploy + + docker: + needs: [jvm, linux, linux-static, mac] + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + runs-on: ubuntu-18.04 + steps: + - name: Git checkout + uses: actions/checkout@v1 + with: + fetch-depth: 1 + submodules: 'true' + + - name: Docker build + env: + DOCKERHUB_USER: "${{ secrets.DOCKERHUB_USER }}" + DOCKERHUB_PASS: "${{ secrets.DOCKERHUB_PASS }}" + run: | + .github/script/docker diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 599a9a69..00000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,176 +0,0 @@ -name: deploy - -# on: -# push: -# tags: -# - "*.*.*" - -on: [push - # , pull_request - ] - -jobs: - uberjar: - runs-on: ubuntu-18.04 - steps: - - name: Git checkout - uses: actions/checkout@v1 - with: - fetch-depth: 1 - submodules: 'true' - - - name: Parse Ref - id: parse-ref - run: | - export VERSION="$(echo $GITHUB_SHA | head -c 7)" - echo "##[set-output name=version;]${VERSION}" - echo "##[set-output name=name;]babashka-${VERSION}" - - - name: Cache deps - uses: actions/cache@v1 - id: cache-deps - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }} # Other deps files? - restore-keys: | - ${{ runner.os }}-maven- - - - name: Fetch deps - if: steps.cache-deps.outputs.cache-hit != 'true' - run: | - lein deps - - - name: Build into uberjar - run: | - echo ${{ steps.parse-ref.outputs.version }} > resources/BABASHKA_VERSION - lein uberjar - - - uses: actions/upload-artifact@v1 - with: - path: target/${{ steps.parse-ref.outputs.name }}-standalone.jar - name: jar - - - name: reflection.json - run: | - lein with-profiles +reflection run - - - uses: actions/upload-artifact@v1 - with: - name: reflection.json - path: reflection.json - - # Initial task to compile a JAR, store as a pipeline artifact to be used by - # downstream builders. - native-image-linux: - needs: [uberjar] - runs-on: ubuntu-18.04 - steps: - - name: Git checkout - uses: actions/checkout@v1 - with: - fetch-depth: 1 - submodules: 'true' - - - uses: actions/download-artifact@v1 - with: - name: jar - path: . - - - uses: actions/download-artifact@v1 - with: - name: reflection.json - path: . - - - name: Parse Ref - id: parse-ref - run: | - export VERSION="$(echo $GITHUB_SHA | head -c 7)" - export BASE=babashka-${VERSION} - echo "##[set-output name=base;]${BASE}" - echo "##[set-output name=name;]${BASE}-linux" - - # Look up at releases here https://github.com/graalvm/graalvm-ce-builds/releases - - name: Prepare GraalVM - uses: DeLaGuardo/setup-graalvm@2.0 - with: - graalvm-version: "20.0.0.java8" - # Newer versions don't work yet - # graalvm-version: "19.3.1.java8" - # graalvm-version: "20.0.0.java8" - - - name: Build Linux native image - run: | - echo "native image at $(which gu)" - export GRAALVM_DEBUG=1 # Seems to work (not ideal, can give runtime errors) - export JAR=${{ steps.parse-ref.outputs.base }}-standalone.jar - export BINARY_NAME=${{ steps.parse-ref.outputs.name }} - script/compile - - - name: Test binary - run: | - export BINARY=./${{ steps.parse-ref.outputs.name }} - $BINARY -e '(println "OK")' - - - run: tar -cvzf ${{ steps.parse-ref.outputs.name }}.tgz ./${{ steps.parse-ref.outputs.name }} - - - uses: actions/upload-artifact@v1 - with: - path: ${{ steps.parse-ref.outputs.name }}.tgz - name: binary-linux - - - # Use GraalVM on macOS to convert JAR to a native macOS binary - native-image-mac: - needs: [uberjar, native-image-linux] # Make it wait for a succesful Linux build as that one runs faster (because of a queue somewhere?). Is also more expensive. - runs-on: macOS-latest - steps: - - name: Git checkout - uses: actions/checkout@v1 - with: - fetch-depth: 1 - submodules: 'true' - - - uses: actions/download-artifact@v1 - with: - name: jar - path: . - - - uses: actions/download-artifact@v1 - with: - name: reflection.json - path: . - - - name: Parse Ref - id: parse-ref - run: | - export VERSION="$(echo $GITHUB_SHA | head -c 7)" - export BASE=babashka-${VERSION} - echo "##[set-output name=base;]${BASE}" - echo "##[set-output name=name;]${BASE}-mac" - - # Look up at releases here https://github.com/graalvm/graalvm-ce-builds/releases - - name: Prepare GraalVM - uses: DeLaGuardo/setup-graalvm@2.0 - with: - graalvm-version: "20.0.0.java8" - # graalvm-version: "19.3.1.java8" - - - name: Build Mac native image - run: | - echo "native image at $(which gu)" - export GRAALVM_DEBUG=1 # Seems to work (not ideal, can give runtime errors) - export JAR=${{ steps.parse-ref.outputs.base }}-standalone.jar - export BINARY_NAME=${{ steps.parse-ref.outputs.name }} - script/compile - - - name: Test binary - run: | - export BINARY=./${{ steps.parse-ref.outputs.name }} - $BINARY -e '(println "OK")' - - - run: tar -cvzf ${{ steps.parse-ref.outputs.name }}.tgz ./${{ steps.parse-ref.outputs.name }} - - - uses: actions/upload-artifact@v1 - with: - path: ${{ steps.parse-ref.outputs.name }}.tgz - name: binary-mac diff --git a/script/compile b/script/compile index 4a1707f7..3d1562a4 100755 --- a/script/compile +++ b/script/compile @@ -2,40 +2,16 @@ set -eo pipefail -GU=`which gu` || true - -if [ -z "$GU" ]; then - if [ -z "$GRAALVM_HOME" ]; then - echo "Please set GRAALVM_HOME" - exit 1 - fi - - GU="$GRAALVM_HOME/bin/gu" -else - GRAALVM_HOME=`echo $GU | sed -e "s/\/bin\/gu$//"` -fi - if [ -z "$BABASHKA_XMX" ]; then export BABASHKA_XMX="-J-Xmx3g" fi -NATIVE_IMAGE=`which native-image` || true - -if [ -z "$NATIVE_IMAGE" ]; then - $GU install native-image || true - - NATIVE_IMAGE=`which native-image` || true - - if [ -z "$NATIVE_IMAGE" ]; then - if [ -z "$GRAALVM_HOME" ]; then - echo "Please set GRAALVM_HOME" - exit 1 - fi - NATIVE_IMAGE="$GRAALVM_HOME/bin/native-image" - fi +if [ -z "$GRAALVM_HOME" ]; then + echo "Please set GRAALVM_HOME" + exit 1 fi -## --- THE ACTUAL SCRIPT --- +$GRAALVM_HOME/bin/gu install native-image BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION) @@ -43,16 +19,16 @@ export JAVA_HOME=$GRAALVM_HOME $GRAALVM_HOME/bin/javac -cp $GRAALVM_HOME/jre/lib/svm/builder/svm.jar resources/CutOffCoreServicesDependencies.java -if [ -z "$JAR" ]; then - lein with-profiles +reflection do run - lein do clean, uberjar - JAR=${JAR:-"target/babashka-$BABASHKA_VERSION-standalone.jar"} +if [ -z "$BABASHKA_JAR" ]; then + lein with-profiles +reflection do run + lein do clean, uberjar + BABASHKA_JAR=${BABASHKA_JAR:-"target/babashka-$BABASHKA_VERSION-standalone.jar"} fi -BINARY_NAME=${BINARY_NAME:-"bb"} +BABASHKA_BINARY=${BABASHKA_BINARY:-"bb"} -args=( -jar $JAR \ - -H:Name=$BINARY_NAME \ +args=( -jar $BABASHKA_JAR \ + -H:Name=$BABASHKA_BINARY \ -H:+ReportExceptionStackTraces \ -J-Dclojure.spec.skip-macros=true \ -J-Dclojure.compiler.direct-linking=true \ @@ -77,10 +53,6 @@ fi $GRAALVM_HOME/bin/native-image "${args[@]}" -LEIN=`which lein` || true - -if [ -z "$LEIN" ]; then - echo "nothing to clean" -else - lein clean -fi \ No newline at end of file +if [ ! -z "$(command -v lein)" ]; then + lein clean +fi diff --git a/.circleci/script/install-clojure b/script/install-clojure similarity index 95% rename from .circleci/script/install-clojure rename to script/install-clojure index b4cb4a55..f4781ba1 100755 --- a/.circleci/script/install-clojure +++ b/script/install-clojure @@ -1,6 +1,6 @@ #!/usr/bin/env bash -install_dir=${1:-/tmp/clojure} +install_dir=${1:-/usr/local} mkdir -p "$install_dir" cd /tmp curl -O -sL https://download.clojure.org/install/clojure-tools-1.10.1.447.tar.gz diff --git a/.circleci/script/install-leiningen b/script/install-leiningen similarity index 50% rename from .circleci/script/install-leiningen rename to script/install-leiningen index fc4b11ea..92a62a41 100755 --- a/.circleci/script/install-leiningen +++ b/script/install-leiningen @@ -1,6 +1,7 @@ #!/usr/bin/env bash curl https://raw.githubusercontent.com/technomancy/leiningen/2.9.1/bin/lein > lein -sudo mkdir -p /usr/local/bin/ -sudo mv lein /usr/local/bin/lein -sudo chmod a+x /usr/local/bin/lein +mkdir -p /usr/local/bin/ +mv lein /usr/local/bin/lein +chmod a+x /usr/local/bin/lein +lein self-install diff --git a/script/test b/script/test index 58f31d5f..d6c8473b 100755 --- a/script/test +++ b/script/test @@ -3,13 +3,16 @@ set -eo pipefail BABASHKA_PRELOADS="" BABASHKA_CLASSPATH="" +echo "running tests part 1" lein test "$@" BABASHKA_PRELOADS='(defn __bb__foo [] "foo") (defn __bb__bar [] "bar")' BABASHKA_PRELOADS_TEST=true +echo "running tests part 2" lein test :only babashka.main-test/preloads-test BABASHKA_PRELOADS="(require '[env-ns])" BABASHKA_CLASSPATH_TEST=true BABASHKA_CLASSPATH="test-resources/babashka/src_for_classpath_test/env" +echo "running tests part 3" lein test :only babashka.classpath-test/classpath-env-test