Github actions enhancements (#319)

.circleci/config.yml

@@ -218,11 +218,11 @@ jobs:
       - run:
           name: Install Clojure
           command: |
-            .circleci/script/install-clojure /usr/local
+            script/install-clojure /usr/local
       - run:
           name: Install Leiningen
           command: |
-            .circleci/script/install-leiningen
+            script/install-leiningen
       - run:
           name: Download GraalVM
           command: |

.github/script/deploy

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [ -z "$GITHUB_HEAD_REF" ] && [ "${GITHUB_REF##*/}" = "master" ]
+then
+    lein deploy clojars
+fi
+
+exit 0;

.github/script/docker

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+image_name="borkdude/babashka"
+image_tag=$(cat resources/BABASHKA_VERSION)
+latest_tag="latest"
+
+if [[ $image_tag =~ SNAPSHOT$ ]]
+then
+    echo "This is a snapshot version"
+    snapshot="true"
+else
+    echo "This is a non-snapshot version"
+    snapshot="false"
+fi
+
+if [ -z "$GITHUB_HEAD_REF" ] && [ "${GITHUB_REF##*/}" = "master" ]
+then
+    echo "Building Docker image $image_name:$image_tag"
+    echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin
+    docker build -t "$image_name" .
+    docker tag "$image_name:$latest_tag" "$image_name:$image_tag"
+    # we only update latest when it's not a SNAPSHOT version
+    if [ "false" = "$snapshot" ]; then
+        echo "Pushing image $image_name:$latest_tag"
+        docker push "$image_name:$latest_tag"
+    fi
+    # we update the version tag, even if it's a SNAPSHOT version
+    echo "Pushing image $image_name:$image_tag"
+    docker push "$image_name:$image_tag"
+else
+    echo "Not publishing Docker image"
+fi
+
+exit 0;

.github/workflows/build.yml

@@ -0,0 +1,345 @@
+name: build
+
+on: [push
+    , pull_request
+    ]
+
+jobs:
+
+  scratch:
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          submodules: 'true'
+
+      - name: Scratch
+        run: |
+          echo "Scratch"
+
+  jvm:
+    # ubuntu 18.04 comes with lein + java8 installed
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          submodules: 'true'
+ 
+      - name: Cache deps
+        uses: actions/cache@v1
+        id: cache-deps
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }}
+          restore-keys: |
+                ${{ runner.os }}-maven-
+
+      - name: Fetch deps
+        if: steps.cache-deps.outputs.cache-hit != 'true'
+        run: |
+          lein deps
+
+      - name: Run tests
+        run: |
+          script/test
+
+      - name: Test libraries
+        run: |
+          sudo script/install-clojure
+          script/run_lib_tests
+
+      - name: Build uberjar
+        run: |
+          lein with-profiles +reflection do run
+          lein do clean, uberjar
+
+      - name: Babashka version
+        id: babashka-version
+        run: |
+          BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION)
+          echo "##[set-output name=version;]${BABASHKA_VERSION}"
+
+      - name: Reflection artifact
+        run: |
+          cp reflection.json babashka-${{ steps.babashka-version.outputs.version }}-reflection.json
+
+      - uses: actions/upload-artifact@v1
+        with:
+          name: jar
+          path: target/babashka-${{ steps.babashka-version.outputs.version }}-standalone.jar
+
+      - uses: actions/upload-artifact@v1
+        with:
+          name: reflection.json
+          path: babashka-${{ steps.babashka-version.outputs.version }}-reflection.json
+
+  linux:
+    needs: [jvm]
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          submodules: 'true'
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: jar
+          path: .
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: reflection.json
+          path: .
+
+      - name: Cache deps
+        uses: actions/cache@v1
+        id: cache-deps
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }}
+          restore-keys: |
+                ${{ runner.os }}-maven-
+
+      - name: Cache GraalVM
+        uses: actions/cache@v1
+        id: cache-graalvm
+        with:
+          path: ~/graalvm-ce-java8-19.3.1
+          key: ${{ runner.os }}-graalvm-19.3.1
+          restore-keys: |
+                ${{ runner.os }}-graalvm-19.3.1
+
+      - name: Download GraalVM
+        run: |
+          cd ~
+          if ! [ -d graalvm-ce-java8-19.3.1 ]; then
+            curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-19.3.1/graalvm-ce-java8-linux-amd64-19.3.1.tar.gz
+            tar xzf graalvm-ce-java8-linux-amd64-19.3.1.tar.gz
+          fi
+
+      - name: Babashka version
+        id: babashka-version
+        run: |
+          BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION)
+          echo "##[set-output name=version;]${BABASHKA_VERSION}"
+
+      - name: Build Linux native image
+        run: |
+          export BABASHKA_JAR=babashka-${{ steps.babashka-version.outputs.version }}-standalone.jar
+          export BABASHKA_XMX="-J-Xmx6g"
+          export GRAALVM_HOME="$HOME/graalvm-ce-java8-19.3.1"
+          cp babashka-${{ steps.babashka-version.outputs.version }}-reflection.json reflection.json
+          script/compile
+
+      - name: Test binary
+        run: |
+          BABASHKA_TEST_ENV=native script/test
+
+      - name: Install clojure
+        run: |
+          sudo script/install-clojure /usr/local
+
+      - name: Test libraries
+        run: |
+          BABASHKA_TEST_ENV=native script/run_lib_tests
+
+      - uses: actions/upload-artifact@v1
+        with:
+          path: bb
+          name: babashka-${{ steps.babashka-version.outputs.version }}-linux-amd64.zip
+
+  linux-static:
+    needs: [jvm]
+    runs-on: ubuntu-16.04
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          submodules: 'true'
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: jar
+          path: .
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: reflection.json
+          path: .
+
+      - name: Cache deps
+        uses: actions/cache@v1
+        id: cache-deps
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }}
+          restore-keys: |
+                ${{ runner.os }}-maven-
+
+      - name: Cache GraalVM
+        uses: actions/cache@v1
+        id: cache-graalvm
+        with:
+          path: ~/graalvm-ce-java8-19.3.1
+          key: ${{ runner.os }}-graalvm-19.3.1
+          restore-keys: |
+                ${{ runner.os }}-graalvm-19.3.1
+
+      - name: Download GraalVM
+        run: |
+          cd ~
+          if ! [ -d graalvm-ce-java8-19.3.1 ]; then
+            curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-19.3.1/graalvm-ce-java8-linux-amd64-19.3.1.tar.gz
+            tar xzf graalvm-ce-java8-linux-amd64-19.3.1.tar.gz
+          fi
+
+      - name: Babashka version
+        id: babashka-version
+        run: |
+          BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION)
+          echo "##[set-output name=version;]${BABASHKA_VERSION}"
+
+      - name: Build Linux native image
+        run: |
+          export BABASHKA_JAR=babashka-${{ steps.babashka-version.outputs.version }}-standalone.jar
+          export BABASHKA_XMX="-J-Xmx6g"
+          export GRAALVM_HOME="$HOME/graalvm-ce-java8-19.3.1"
+          export BABASHKA_STATIC=true
+          cp babashka-${{ steps.babashka-version.outputs.version }}-reflection.json reflection.json
+          script/compile
+
+      - name: Test binary
+        run: |
+          ./bb '(+ 1 2 3)'
+          BABASHKA_TEST_ENV=native script/test
+
+      - name: Install clojure
+        run: |
+          sudo script/install-clojure
+
+      - name: Test libraries
+        run: |
+          BABASHKA_TEST_ENV=native script/run_lib_tests
+
+      - uses: actions/upload-artifact@v1
+        with:
+          path: bb
+          name: babashka-${{ steps.babashka-version.outputs.version }}-linux-static-amd64.zip
+
+  mac:
+    needs: [jvm]
+    runs-on: macOS-latest
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          submodules: 'true'
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: jar
+          path: .
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: reflection.json
+          path: .
+
+      - name: Cache GraalVM
+        uses: actions/cache@v1
+        id: cache-graalvm
+        with:
+          path: ~/graalvm-ce-java8-19.3.1
+          key: ${{ runner.os }}-graalvm-19.3.1
+          restore-keys: |
+                ${{ runner.os }}-graalvm-19.3.1
+
+      - name: Download GraalVM
+        run: |
+          cd ~
+          if ! [ -d graalvm-ce-java8-19.3.1 ]; then
+            curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-19.3.1/graalvm-ce-java8-darwin-amd64-19.3.1.tar.gz
+            tar xzf graalvm-ce-java8-darwin-amd64-19.3.1.tar.gz
+          fi
+
+      - name: Babashka version
+        id: babashka-version
+        run: |
+          BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION)
+          echo "##[set-output name=version;]${BABASHKA_VERSION}"
+
+      - name: Build macOS native image
+        run: |
+          export BABASHKA_JAR=babashka-${{ steps.babashka-version.outputs.version }}-standalone.jar
+          export BABASHKA_XMX="-J-Xmx6g"
+          export GRAALVM_HOME="$HOME/graalvm-ce-java8-19.3.1/Contents/Home"
+          cp babashka-${{ steps.babashka-version.outputs.version }}-reflection.json reflection.json
+          script/compile
+
+      - name: Test binary
+        run: |
+          sudo script/install-leiningen
+          BABASHKA_TEST_ENV=native script/test
+
+      - name: Test libraries
+        run: |
+          sudo script/install-clojure
+          BABASHKA_TEST_ENV=native script/run_lib_tests
+
+      - uses: actions/upload-artifact@v1
+        with:
+          path: bb
+          name: babashka-${{ steps.babashka-version.outputs.version }}-macos-amd64.zip
+
+  deploy:
+    needs: [jvm, linux, linux-static, mac]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          submodules: 'true'
+
+      - name: Cache deps
+        uses: actions/cache@v1
+        id: cache-deps
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }}
+          restore-keys: |
+                ${{ runner.os }}-maven-
+
+      - name: Deploy
+        env:
+          CLOJARS_USER: "${{ secrets.CLOJARS_USER }}"
+          CLOJARS_PASS: "${{ secrets.CLOJARS_PASS }}"
+        run: |
+          .github/script/deploy
+
+  docker:
+    needs: [jvm, linux, linux-static, mac]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          submodules: 'true'
+
+      - name: Docker build
+        env:
+          DOCKERHUB_USER: "${{ secrets.DOCKERHUB_USER }}"
+          DOCKERHUB_PASS: "${{ secrets.DOCKERHUB_PASS }}"
+        run: |
+          .github/script/docker

.github/workflows/deploy.yml

@@ -1,176 +0,0 @@
-name: deploy
-
-# on:
-#   push:
-#     tags:
-#       - "*.*.*"
-
-on: [push
-   # , pull_request
-   ]
-
-jobs:
-  uberjar:
-    runs-on: ubuntu-18.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v1
-        with:
-          fetch-depth: 1
-          submodules: 'true'
-          
-      - name: Parse Ref
-        id: parse-ref
-        run: |
-          export VERSION="$(echo $GITHUB_SHA | head -c 7)"
-          echo "##[set-output name=version;]${VERSION}"              
-          echo "##[set-output name=name;]babashka-${VERSION}"
-
-      - name: Cache deps
-        uses: actions/cache@v1
-        id: cache-deps
-        with: 
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('project.clj') }} # Other deps files?
-          restore-keys: |
-                ${{ runner.os }}-maven-
-  
-      - name: Fetch deps
-        if: steps.cache-deps.outputs.cache-hit != 'true'
-        run: |
-          lein deps
-
-      - name: Build into uberjar
-        run: |
-          echo ${{ steps.parse-ref.outputs.version }} > resources/BABASHKA_VERSION
-          lein uberjar
-          
-      - uses: actions/upload-artifact@v1
-        with:
-          path: target/${{ steps.parse-ref.outputs.name }}-standalone.jar
-          name: jar
-          
-      - name: reflection.json
-        run: |
-          lein with-profiles +reflection run
-      
-      - uses: actions/upload-artifact@v1
-        with:
-          name: reflection.json
-          path: reflection.json
-                
-  # Initial task to compile a JAR, store as a pipeline artifact to be used by
-  # downstream builders.
-  native-image-linux:
-    needs: [uberjar]
-    runs-on: ubuntu-18.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v1
-        with:
-          fetch-depth: 1
-          submodules: 'true'
-          
-      - uses: actions/download-artifact@v1
-        with:
-          name: jar
-          path: .
-          
-      - uses: actions/download-artifact@v1
-        with:
-          name: reflection.json
-          path: .                   
-
-      - name: Parse Ref
-        id: parse-ref
-        run: |
-          export VERSION="$(echo $GITHUB_SHA | head -c 7)" 
-          export BASE=babashka-${VERSION}
-          echo "##[set-output name=base;]${BASE}"
-          echo "##[set-output name=name;]${BASE}-linux"
-          
-      # Look up at releases here https://github.com/graalvm/graalvm-ce-builds/releases
-      - name: Prepare GraalVM
-        uses: DeLaGuardo/setup-graalvm@2.0
-        with:
-          graalvm-version: "20.0.0.java8"  
-          # Newer versions don't work yet        
-          # graalvm-version: "19.3.1.java8"
-          # graalvm-version: "20.0.0.java8"
-            
-      - name: Build Linux native image
-        run: |
-          echo "native image at $(which gu)" 
-          export GRAALVM_DEBUG=1 # Seems to work (not ideal, can give runtime errors)
-          export JAR=${{ steps.parse-ref.outputs.base }}-standalone.jar
-          export BINARY_NAME=${{ steps.parse-ref.outputs.name }}          
-          script/compile
-          
-      - name: Test binary
-        run: |
-          export BINARY=./${{ steps.parse-ref.outputs.name }}
-          $BINARY -e '(println "OK")'
-          
-      - run: tar -cvzf ${{ steps.parse-ref.outputs.name }}.tgz ./${{ steps.parse-ref.outputs.name }}
-
-      - uses: actions/upload-artifact@v1
-        with:
-          path: ${{ steps.parse-ref.outputs.name }}.tgz
-          name: binary-linux
-          
-          
-  # Use GraalVM on macOS to convert JAR to a native macOS binary
-  native-image-mac:
-    needs: [uberjar, native-image-linux] # Make it wait for a succesful Linux build as that one runs faster (because of a queue somewhere?). Is also more expensive.
-    runs-on: macOS-latest
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v1
-        with:
-          fetch-depth: 1
-          submodules: 'true'
-          
-      - uses: actions/download-artifact@v1
-        with:
-          name: jar
-          path: .  
-          
-      - uses: actions/download-artifact@v1
-        with:
-          name: reflection.json
-          path: .     
-
-      - name: Parse Ref
-        id: parse-ref
-        run: |
-          export VERSION="$(echo $GITHUB_SHA | head -c 7)" 
-          export BASE=babashka-${VERSION}
-          echo "##[set-output name=base;]${BASE}"
-          echo "##[set-output name=name;]${BASE}-mac"
-      
-      # Look up at releases here https://github.com/graalvm/graalvm-ce-builds/releases
-      - name: Prepare GraalVM
-        uses: DeLaGuardo/setup-graalvm@2.0
-        with:
-          graalvm-version: "20.0.0.java8"
-          # graalvm-version: "19.3.1.java8"
-            
-      - name: Build Mac native image
-        run: |
-          echo "native image at $(which gu)" 
-          export GRAALVM_DEBUG=1 # Seems to work (not ideal, can give runtime errors)
-          export JAR=${{ steps.parse-ref.outputs.base }}-standalone.jar
-          export BINARY_NAME=${{ steps.parse-ref.outputs.name }}          
-          script/compile
-          
-      - name: Test binary
-        run: |
-          export BINARY=./${{ steps.parse-ref.outputs.name }}
-          $BINARY -e '(println "OK")'
-          
-      - run: tar -cvzf ${{ steps.parse-ref.outputs.name }}.tgz ./${{ steps.parse-ref.outputs.name }}
-
-      - uses: actions/upload-artifact@v1
-        with:
-          path: ${{ steps.parse-ref.outputs.name }}.tgz
-          name: binary-mac

script/compile

@@ -2,40 +2,16 @@
 
 set -eo pipefail
 
-GU=`which gu` || true
-
-if [ -z "$GU" ]; then
-  if [ -z "$GRAALVM_HOME" ]; then
-      echo "Please set GRAALVM_HOME"
-      exit 1
-  fi
-
-  GU="$GRAALVM_HOME/bin/gu"
-else
-  GRAALVM_HOME=`echo $GU | sed -e "s/\/bin\/gu$//"`
-fi
-
 if [ -z "$BABASHKA_XMX" ]; then
     export BABASHKA_XMX="-J-Xmx3g"
 fi
 
-NATIVE_IMAGE=`which native-image` || true
+if [ -z "$GRAALVM_HOME" ]; then
-
+    echo "Please set GRAALVM_HOME"
-if [ -z "$NATIVE_IMAGE" ]; then
+    exit 1
-  $GU install native-image || true
-
-  NATIVE_IMAGE=`which native-image` || true
-
-  if [ -z "$NATIVE_IMAGE" ]; then
-    if [ -z "$GRAALVM_HOME" ]; then
-        echo "Please set GRAALVM_HOME"
-        exit 1
-    fi
-    NATIVE_IMAGE="$GRAALVM_HOME/bin/native-image"
-  fi
 fi
 
-## --- THE ACTUAL SCRIPT ---
+$GRAALVM_HOME/bin/gu install native-image
 
 BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION)
 
@@ -43,16 +19,16 @@ export JAVA_HOME=$GRAALVM_HOME
 
 $GRAALVM_HOME/bin/javac -cp $GRAALVM_HOME/jre/lib/svm/builder/svm.jar resources/CutOffCoreServicesDependencies.java
 
-if [ -z "$JAR" ]; then
+if [ -z "$BABASHKA_JAR" ]; then
-  lein with-profiles +reflection do run
+   lein with-profiles +reflection do run
-  lein do clean, uberjar
+   lein do clean, uberjar
-  JAR=${JAR:-"target/babashka-$BABASHKA_VERSION-standalone.jar"}
+  BABASHKA_JAR=${BABASHKA_JAR:-"target/babashka-$BABASHKA_VERSION-standalone.jar"}
 fi
 
-BINARY_NAME=${BINARY_NAME:-"bb"}
+BABASHKA_BINARY=${BABASHKA_BINARY:-"bb"}
 
-args=( -jar $JAR \
+args=( -jar $BABASHKA_JAR \
-            -H:Name=$BINARY_NAME \
+            -H:Name=$BABASHKA_BINARY \
             -H:+ReportExceptionStackTraces \
             -J-Dclojure.spec.skip-macros=true \
             -J-Dclojure.compiler.direct-linking=true \
@@ -77,10 +53,6 @@ fi
 
 $GRAALVM_HOME/bin/native-image "${args[@]}"
 
-LEIN=`which lein` || true
+if [ ! -z "$(command -v lein)" ]; then
-
+    lein clean
-if [ -z "$LEIN" ]; then
-  echo "nothing to clean"
-else
-  lein clean
 fi

script/install-clojure

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-install_dir=${1:-/tmp/clojure}
+install_dir=${1:-/usr/local}
 mkdir -p "$install_dir"
 cd /tmp
 curl -O -sL https://download.clojure.org/install/clojure-tools-1.10.1.447.tar.gz

script/install-leiningen

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 curl https://raw.githubusercontent.com/technomancy/leiningen/2.9.1/bin/lein > lein
-sudo mkdir -p /usr/local/bin/
+mkdir -p /usr/local/bin/
-sudo mv lein /usr/local/bin/lein
+mv lein /usr/local/bin/lein
-sudo chmod a+x /usr/local/bin/lein
+chmod a+x /usr/local/bin/lein
+lein self-install

script/test

@@ -3,13 +3,16 @@
 set -eo pipefail
 BABASHKA_PRELOADS=""
 BABASHKA_CLASSPATH=""
+echo "running tests part 1"
 lein test "$@"
 
 BABASHKA_PRELOADS='(defn __bb__foo [] "foo") (defn __bb__bar [] "bar")'
 BABASHKA_PRELOADS_TEST=true
+echo "running tests part 2"
 lein test :only babashka.main-test/preloads-test
 
 BABASHKA_PRELOADS="(require '[env-ns])"
 BABASHKA_CLASSPATH_TEST=true
 BABASHKA_CLASSPATH="test-resources/babashka/src_for_classpath_test/env"
+echo "running tests part 3"
 lein test :only babashka.classpath-test/classpath-env-test