luciano/babashka
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use dynamic configuration to short CI (#1294)

Browse source
This commit is contained in:
agata-anastazja 2022-06-20 19:03:03 +01:00 committed by GitHub
parent 7769675648
commit 381b9f0a4f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 268 additions and 506 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.build/bb.edn
View file

@ -2,10 +2,5 @@
:deps {borkdude/gh-release-artifact
#_{:local/root "../gh-release-artifact"}
{:git/url "https://github.com/borkdude/gh-release-artifact"
:sha "cf082df46a648178d1904e9cbcb787d8136a35c6"}
babashka/.build {:git/url "https://github.com/babashka/.build"
:git/sha "1ce87f4960fea14f4d1248052e585e6027555d33"}}
:tasks {:requires ([scripts.short-ci :as short])
release-artifact babashka.release-artifact/release
short-ci {:task (do (alter-var-root (var short/config) assoc :skip-if-only [(re-pattern ".*")])
(apply short/main *command-line-args*))}}}
:sha "cf082df46a648178d1904e9cbcb787d8136a35c6"}}
:tasks {release-artifact babashka.release-artifact/release}}

515
.circleci/config.yml
View file

@ -3,17 +3,17 @@
# Check https://circleci.com/docs/2.0/language-clojure/ for more details
#
version: 2.1
commands:
setup-docker-buildx:
steps:
- run:
name: Create multi-platform capabale buildx builder
command: |
docker run --privileged --rm tonistiigi/binfmt --install all
docker buildx create --name ci-builder --use
# this allows you to use CircleCI's dynamic configuration feature
setup: true
# the continuation orb is required in order to use dynamic configuration
orbs:
continuation: circleci/continuation@0.1.2
# our defined job, and its steps
jobs:
short-if-irrelevant:
setup:
docker:
- image: cimg/clojure:1.11.1
steps:
@ -27,497 +27,14 @@ jobs:
name: Rename bb binary
command: mv /tmp/bb /tmp/bbb
- run:
name: Short CI if only irrelevant changes
name: Generate config
command: |
/tmp/bbb --config .build/bb.edn short-ci "circleci-agent step halt"
jvm:
docker:
- image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye
working_directory: ~/repo
environment:
LEIN_ROOT: "true"
BABASHKA_PLATFORM: linux # could be used in jar name
resource_class: large
steps:
- checkout
- run:
name: "Pull Submodules"
command: |
git submodule init
git submodule update
- restore_cache:
keys:
- v1-dependencies-{{ checksum "project.clj" }}-{{ checksum "deps.edn" }}
# fallback to using the latest cache if no exact match is found
- v1-dependencies-
- run:
name: Install Clojure
command: |
sudo script/install-clojure
- run:
name: Run JVM tests
command: |
export BABASHKA_FEATURE_JDBC=true
export BABASHKA_FEATURE_POSTGRESQL=true
script/test
script/run_lib_tests
- run:
name: Run as lein command
command: |
.circleci/script/lein
- run:
name: Create uberjar
command: |
mkdir -p /tmp/release
script/uberjar
VERSION=$(cat resources/BABASHKA_VERSION)
jar=target/babashka-$VERSION-standalone.jar
cp $jar /tmp/release
java -jar $jar script/reflection.clj
reflection="babashka-$VERSION-reflection.json"
java -jar "$jar" --config .build/bb.edn --deps-root . release-artifact "$jar"
java -jar "$jar" --config .build/bb.edn --deps-root . release-artifact "$reflection"
- store_artifacts:
path: /tmp/release
destination: release
- save_cache:
paths:
- ~/.m2
key: v1-dependencies-{{ checksum "project.clj" }}-{{ checksum "deps.edn" }}
linux:
docker:
- image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye
working_directory: ~/repo
environment:
LEIN_ROOT: "true"
GRAALVM_VERSION: "22.1.0"
GRAALVM_HOME: /home/circleci/graalvm-ce-java11-22.1.0
BABASHKA_PLATFORM: linux # used in release script
BABASHKA_TEST_ENV: native
BABASHKA_XMX: "-J-Xmx6500m"
resource_class: large
steps:
- checkout
- run:
name: "Pull Submodules"
command: |
git submodule init
git submodule update
- restore_cache:
keys:
- linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- run:
name: Install Clojure
command: |
sudo script/install-clojure
- run:
name: Install native dev tools
command: |
sudo apt-get update
sudo apt-get -y install build-essential zlib1g-dev
- run:
name: Download GraalVM
command: |
script/install-graalvm
- run:
name: Build binary
command: |
script/uberjar
script/compile
no_output_timeout: 30m
- run:
name: Run tests
command: |
script/test
script/run_lib_tests
- run:
name: Release
command: |
.circleci/script/release
- persist_to_workspace:
root: /tmp
paths:
- release
- save_cache:
paths:
- ~/.m2
- ~/graalvm-ce-java11-22.1.0
key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- store_artifacts:
path: /tmp/release
destination: release
- run:
name: Publish artifact link to Slack
command: |
./bb .circleci/script/publish_artifact.clj || true
linux-static:
docker:
- image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye
working_directory: ~/repo
environment:
LEIN_ROOT: "true"
GRAALVM_VERSION: "22.1.0"
GRAALVM_HOME: /home/circleci/graalvm-ce-java11-22.1.0
BABASHKA_PLATFORM: linux # used in release script
BABASHKA_TEST_ENV: native
BABASHKA_STATIC: "true"
BABASHKA_MUSL: "true"
BABASHKA_XMX: "-J-Xmx6500m"
resource_class: large
steps:
- checkout
- attach_workspace:
at: /tmp
- run:
name: "Pull Submodules"
command: |
git submodule init
git submodule update
# - run:
# name: "Short circuit on SNAPSHOT"
# command: |
# VERSION=$(cat resources/BABASHKA_VERSION)
# if [[ "$VERSION" == *-SNAPSHOT ]]
# then
# circleci task halt
# fi
- restore_cache:
keys:
- linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- run:
name: Install Clojure
command: |
sudo script/install-clojure
- run:
name: Install native dev tools
command: |
sudo apt-get update
sudo apt-get -y install build-essential zlib1g-dev
sudo -E script/setup-musl
- run:
name: Download GraalVM
command: |
script/install-graalvm
- run:
name: Build binary
command: |
script/uberjar
script/compile
no_output_timeout: 30m
- run:
name: Run tests
command: |
script/test
script/run_lib_tests
- run:
name: Release
command: |
.circleci/script/release
- persist_to_workspace:
root: /tmp
paths:
- release
- save_cache:
paths:
- ~/.m2
- ~/graalvm-ce-java11-22.1.0
key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- store_artifacts:
path: /tmp/release
destination: release
- run:
name: Publish artifact link to Slack
command: |
./bb .circleci/script/publish_artifact.clj || true
linux-aarch64:
machine:
image: ubuntu-2004:202111-01
resource_class: arm.large
working_directory: ~/repo
environment:
LEIN_ROOT: "true"
GRAALVM_VERSION: "22.1.0"
GRAALVM_HOME: /home/circleci/graalvm-ce-java11-22.1.0
BABASHKA_PLATFORM: linux # used in release script
BABASHKA_ARCH: aarch64
BABASHKA_TEST_ENV: native
BABASHKA_XMX: "-J-Xmx6500m"
steps:
- checkout
- run:
name: "Pull Submodules"
command: |
git submodule init
git submodule update
- run:
name: Install Clojure
command: |
sudo script/install-clojure
- restore_cache:
keys:
- linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- run:
name: Install native dev tools
command: |
sudo apt-get update
sudo apt-get -y install build-essential zlib1g-dev
- run:
name: Download GraalVM
command: |
script/install-graalvm
- run:
name: Build binary
command: |
script/uberjar
script/compile
no_output_timeout: 30m
- run:
name: Run tests
command: |
script/test
script/run_lib_tests
- run:
name: Release
command: |
.circleci/script/release
- persist_to_workspace:
root: /tmp
paths:
- release
- save_cache:
paths:
- ~/.m2
- ~/graalvm-ce-java11-22.1.0
key: linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- store_artifacts:
path: /tmp/release
destination: release
- run:
name: Publish artifact link to Slack
command: |
./bb .circleci/script/publish_artifact.clj || true
linux-aarch64-static:
machine:
image: ubuntu-2004:202111-01
resource_class: arm.large
working_directory: ~/repo
environment:
LEIN_ROOT: "true"
GRAALVM_VERSION: "22.1.0"
GRAALVM_HOME: /home/circleci/graalvm-ce-java11-22.1.0
BABASHKA_PLATFORM: linux # used in release script
BABASHKA_ARCH: aarch64
BABASHKA_TEST_ENV: native
BABASHKA_XMX: "-J-Xmx6500m"
BABASHKA_STATIC: "true"
steps:
- checkout
- run:
name: "Pull Submodules"
command: |
git submodule init
git submodule update
# - run:
# name: "Short circuit on SNAPSHOT"
# command: |
# VERSION=$(cat resources/BABASHKA_VERSION)
# if [[ "$VERSION" == *-SNAPSHOT ]]
# then
# circleci task halt
# fi
- run:
name: Install Clojure
command: |
sudo script/install-clojure
- restore_cache:
keys:
- linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- run:
name: Install native dev tools
command: |
sudo apt-get update
sudo apt-get -y install build-essential zlib1g-dev
# sudo -E script/setup-musl
- run:
name: Download GraalVM
command: |
script/install-graalvm
- run:
name: Build binary
command: |
script/uberjar
script/compile
no_output_timeout: 30m
- run:
name: Run tests
command: |
script/test
script/run_lib_tests
- run:
name: Release
command: |
.circleci/script/release
- persist_to_workspace:
root: /tmp
paths:
- release
- save_cache:
paths:
- ~/.m2
- ~/graalvm-ce-java11-22.1.0
key: linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- store_artifacts:
path: /tmp/release
destination: release
- run:
name: Publish artifact link to Slack
command: |
./bb .circleci/script/publish_artifact.clj || true
mac:
macos:
xcode: "12.0.0"
environment:
MACOSX_DEPLOYMENT_TARGET: 10.13 # 10.12 is EOL
GRAALVM_VERSION: "22.1.0"
GRAALVM_HOME: /Users/distiller/graalvm-ce-java11-22.1.0/Contents/Home
BABASHKA_PLATFORM: macos # used in release script
BABASHKA_TEST_ENV: native
BABASHKA_XMX: "-J-Xmx6500m"
resource_class: large
steps:
- checkout
- run:
name: "Pull Submodules"
command: |
git submodule init
git submodule update
- restore_cache:
keys:
- mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- run:
name: Install Clojure
command: |
script/install-clojure
- run:
name: Install Leiningen
command: |
script/install-leiningen
- run:
name: Download GraalVM
command: |
script/install-graalvm
- run:
name: Build binary
command: |
export PATH=$GRAALVM_HOME/bin:$PATH
script/uberjar
script/compile
no_output_timeout: 30m
- run:
name: Run tests
command: |
export PATH=$GRAALVM_HOME/bin:$PATH
script/test
script/run_lib_tests
- run:
name: Release
command: |
.circleci/script/release
- save_cache:
paths:
- ~/.m2
- ~/graalvm-ce-java11-22.1.0/Contents/Home
key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
- store_artifacts:
path: /tmp/release
destination: release
- run:
name: Publish artifact link to Slack
command: |
./bb .circleci/script/publish_artifact.clj || true
deploy:
resource_class: large
docker:
- image: circleci/clojure:lein-2.9.8
working_directory: ~/repo
environment:
LEIN_ROOT: "true"
steps:
- checkout
- run:
name: "Pull Submodules"
command: |
git submodule init
git submodule update
- restore_cache:
keys:
- v1-dependencies-{{ checksum "project.clj" }}
# fallback to using the latest cache if no exact match is found
- v1-dependencies-
- run: .circleci/script/deploy
- save_cache:
paths:
- ~/.m2
key: v1-dependencies-{{ checksum "project.clj" }}
docker:
machine:
image: ubuntu-2004:202111-01
steps:
- checkout
- run:
name: "Pull Submodules"
command: |
git submodule init
git submodule update
- setup-docker-buildx
- attach_workspace:
at: /tmp
- run:
name: Build uberjar
command: script/uberjar
- run:
name: Build Docker image
environment:
PLATFORMS: linux/amd64,linux/arm64
command: java -jar ./target/babashka-$(cat resources/BABASHKA_VERSION)-standalone.jar .circleci/script/docker.clj
/tmp/bbb .circleci/script/short_ci.clj > generated_config.yml
- continuation/continue:
configuration_path: generated_config.yml
# our single workflow, that triggers the setup job defined above
workflows:
version: 2
ci:
setup:
jobs:
- short-if-irrelevant
- jvm:
requires:
- short-if-irrelevant
- linux:
requires:
- short-if-irrelevant
- linux-static:
requires:
- short-if-irrelevant
- mac:
requires:
- short-if-irrelevant
- linux-aarch64:
requires:
- short-if-irrelevant
- linux-aarch64-static:
requires:
- short-if-irrelevant
- deploy:
filters:
branches:
only: master
requires:
- jvm
- linux
# - mac # TODO: Enable when not needed from Actions
- docker:
filters:
branches:
only: master
requires:
- linux
- linux-static
- linux-aarch64
- setup

250
.circleci/script/short_ci.clj Normal file
View file

@ -0,0 +1,250 @@
(ns short-ci
(:require
[babashka.tasks :as tasks]
[clj-yaml.core :as yaml]
[clojure.string :as str]
[flatland.ordered.map :refer [ordered-map]]))
(defn run
([cmd-name cmd]
(run cmd-name cmd nil))
([cmd-name cmd no-output-timeout]
(let [base {:run {:name cmd-name
:command cmd}}]
(if no-output-timeout
(assoc-in base [:run :no_output_timeout] no-output-timeout)
base))))
(defn gen-steps
[shorted? steps]
(if shorted?
[(run "Shorted" "echo 'Skipping Run'")]
steps))
(defn pull-submodules
[]
(run "Pull Submodules" "git submodule init\ngit submodule update"))
(defn deploy
[shorted?]
(ordered-map
:resource_class "large"
:docker [{:image "circleci/clojure:lein-2.9.8"}]
:working_directory "~/repo"
:environment {:LEIN_ROOT "true"}
:steps (gen-steps
shorted?
[:checkout
(pull-submodules)
{:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}"
"v1-dependencies-"]}}
{:run ".circleci/script/deploy"}
{:save_cache {:paths ["~/.m2"]
:key "v1-dependencies-{{ checksum \"project.clj\" }}"}}])))
(defn docker
[shorted?]
(ordered-map
:machine {:image "ubuntu-2004:202111-01"}
:steps
(gen-steps
shorted?
[:checkout
(pull-submodules)
"setup-docker-buildx"
{:attach_workspace {:at "/tmp"}}
(run "Build uberjar" "script/uberjar")
{:run
{:name "Build Docker image"
:environment {:PLATFORMS "linux/amd64,linux/arm64"}
:command
"java -jar ./target/babashka-$(cat resources/BABASHKA_VERSION)-standalone.jar .circleci/script/docker.clj"}}])))
(defn jvm
[shorted?]
(ordered-map
:docker [{:image "circleci/clojure:openjdk-11-lein-2.9.8-bullseye"}]
:working_directory "~/repo"
:environment {:LEIN_ROOT "true"
:BABASHKA_PLATFORM "linux"}
:resource_class "large"
:steps
(gen-steps
shorted?
[:checkout
(pull-submodules)
{:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}"
"v1-dependencies-"]}}
(run "Install Clojure" "sudo script/install-clojure")
(run
"Run JVM tests"
"export BABASHKA_FEATURE_JDBC=true
export BABASHKA_FEATURE_POSTGRESQL=true
script/test\nscript/run_lib_tests")
(run "Run as lein command" ".circleci/script/lein")
(run
"Create uberjar"
"mkdir -p /tmp/release
script/uberjar
VERSION=$(cat resources/BABASHKA_VERSION)
jar=target/babashka-$VERSION-standalone.jar
cp $jar /tmp/release
java -jar $jar script/reflection.clj
reflection=\"babashka-$VERSION-reflection.json\"
java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$jar\"
java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$reflection\"")
{:store_artifacts {:path "/tmp/release"
:destination "release"}}
{:save_cache {:paths ["~/.m2"]
:key "v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}"}}])))
(defn unix
[shorted? static? musl? arch executor-conf resource-class graalvm-home platform]
(let [env {:LEIN_ROOT "true"
:GRAALVM_VERSION "22.1.0"
:GRAALVM_HOME graalvm-home
:BABASHKA_PLATFORM (if (= "mac" platform)
"macos"
platform)
:BABASHKA_TEST_ENV "native"
:BABASHKA_ARCH arch
:BABASHKA_XMX "-J-Xmx6500m"}
env (if static?
(assoc env :BABASHKA_STATIC "true")
env)
env (if musl?
(assoc env :BABASHKA_MUSL "true")
env)
env (if (= "mac" platform)
(assoc env :MACOSX_DEPLOYMENT_TARGET 10.13)
env)
base-install-cmd "sudo apt-get update\nsudo apt-get -y install build-essential zlib1g-dev"
cache-key (format "%s-%s{{ checksum \"project.clj\" }}-{{ checksum \".circleci/config.yml\" }}"
platform
(if (= "aarch64" arch)
"aarch64-"
""))]
(merge
executor-conf
(ordered-map
:working_directory "~/repo"
:environment env
:resource_class resource-class
:steps (gen-steps shorted?
(filter some?
[:checkout
{:attach_workspace {:at "/tmp"}}
(run "Pull Submodules" "git submodule init\ngit submodule update")
{:restore_cache
{:keys [cache-key]}}
(run "Install Clojure" "sudo script/install-clojure")
(when (= "mac" platform)
(run "Install Leiningen" "script/install-leiningen"))
(when (not= "mac" platform)
(run "Install native dev tools"
(if (and static? musl? (not= "aarch64" arch))
(str base-install-cmd "\nsudo -E script/setup-musl")
base-install-cmd)))
(run "Download GraalVM" "script/install-graalvm")
(run "Build binary" "script/uberjar\nscript/compile" "30m")
(run "Run tests" "script/test\nscript/run_lib_tests")
(run "Release" ".circleci/script/release")
{:persist_to_workspace {:root "/tmp"
:paths ["release"]}}
{:save_cache
{:paths ["~/.m2" "~/graalvm-ce-java11-22.1.0"]
:key cache-key}}
{:store_artifacts {:path "/tmp/release"
:destination "release"}}
(run "Publish artifact link to Slack"
"./bb .circleci/script/publish_artifact.clj || true")]))))))
(defn make-config
[shorted?]
(let [docker-executor-conf {:docker [{:image "circleci/clojure:openjdk-11-lein-2.9.8-bullseye"}]}
machine-executor-conf {:machine {:image "ubuntu-2004:202111-01"}}
mac-executor-conf {:macos {:xcode "12.0.0"}}
linux-graalvm-home "/home/circleci/graalvm-ce-java11-22.1.0"
mac-graalvm-home "/Users/distiller/graalvm-ce-java11-22.1.0/Contents/Home"]
(ordered-map
:version 2.1
:commands
{:setup-docker-buildx
{:steps
[{:run
{:name "Create multi-platform capabale buildx builder"
:command
"docker run --privileged --rm tonistiigi/binfmt --install all\ndocker buildx create --name ci-builder --use"}}]}}
:jobs (ordered-map
:jvm (jvm shorted?)
:linux (unix shorted? false false "x86_64" docker-executor-conf "large" linux-graalvm-home "linux")
:linux-static
(unix shorted? true true "x86_64" docker-executor-conf "large" linux-graalvm-home "linux")
:linux-aarch64 (unix shorted?
false
false
"aarch64"
machine-executor-conf
"arm.large"
linux-graalvm-home
"linux")
:linux-aarch64-static
(unix shorted? true false "aarch64" machine-executor-conf "arm.large" linux-graalvm-home "linux")
:mac (unix shorted? false false "x86_64" mac-executor-conf "large" mac-graalvm-home "mac")
:deploy (deploy shorted?)
:docker (docker shorted?))
:workflows (ordered-map
:version 2
:ci {:jobs ["jvm"
"linux"
"linux-static"
"mac"
"linux-aarch64"
"linux-aarch64-static"
{:deploy {:filters {:branches {:only "master"}}
:requires ["jvm" "linux"]}}
{:docker {:filters {:branches {:only "master"}}
:requires ["linux" "linux-static" "linux-aarch64"]}}]}))))
(def skip-config
{:skip-if-only [#".*.md$"]})
(defn get-changes
[]
(-> (tasks/shell {:out :string} "git diff --name-only HEAD~1")
(:out)
(str/split-lines)))
(defn irrelevant-change?
[change regexes]
(some? (some #(re-matches % change) regexes)))
(defn relevant?
[change-set regexes]
(some? (some #(not (irrelevant-change? % regexes)) change-set)))
(defn main
[]
(let [{:keys [skip-if-only]} skip-config
changed-files (get-changes)
conf (make-config (not (relevant? changed-files skip-if-only)))]
(println (yaml/generate-string conf
:dumper-options
{:flow-style :block}))))
(when (= *file* (System/getProperty "babashka.file"))
(main))
(comment
(main)
(def regexes
[#".*.md$"
#".*.clj$"]) ; ignore clojure files
(:out (tasks/shell {:out :string} "ls"))
(irrelevant-change? "src/file.png" regexes)
(re-matches #".*.clj$" "src/file.clj.dfff")
(relevant? ["src/file.clj"] regexes))