From 381b9f0a4f12bc47097c4d26a9b475bcf67f0a62 Mon Sep 17 00:00:00 2001 From: agata-anastazja Date: Mon, 20 Jun 2022 19:03:03 +0100 Subject: [PATCH] use dynamic configuration to short CI (#1294) --- .build/bb.edn | 9 +- .circleci/config.yml | 515 ++-------------------------------- .circleci/script/short_ci.clj | 250 +++++++++++++++++ 3 files changed, 268 insertions(+), 506 deletions(-) create mode 100644 .circleci/script/short_ci.clj diff --git a/.build/bb.edn b/.build/bb.edn index 13e47f4a..c1e5ff3e 100644 --- a/.build/bb.edn +++ b/.build/bb.edn @@ -2,10 +2,5 @@ :deps {borkdude/gh-release-artifact #_{:local/root "../gh-release-artifact"} {:git/url "https://github.com/borkdude/gh-release-artifact" - :sha "cf082df46a648178d1904e9cbcb787d8136a35c6"} - babashka/.build {:git/url "https://github.com/babashka/.build" - :git/sha "1ce87f4960fea14f4d1248052e585e6027555d33"}} - :tasks {:requires ([scripts.short-ci :as short]) - release-artifact babashka.release-artifact/release - short-ci {:task (do (alter-var-root (var short/config) assoc :skip-if-only [(re-pattern ".*")]) - (apply short/main *command-line-args*))}}} + :sha "cf082df46a648178d1904e9cbcb787d8136a35c6"}} + :tasks {release-artifact babashka.release-artifact/release}} diff --git a/.circleci/config.yml b/.circleci/config.yml index f156f337..3e0293e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,17 +3,17 @@ # Check https://circleci.com/docs/2.0/language-clojure/ for more details # version: 2.1 -commands: - setup-docker-buildx: - steps: - - run: - name: Create multi-platform capabale buildx builder - command: | - docker run --privileged --rm tonistiigi/binfmt --install all - docker buildx create --name ci-builder --use +# this allows you to use CircleCI's dynamic configuration feature +setup: true + +# the continuation orb is required in order to use dynamic configuration +orbs: + continuation: circleci/continuation@0.1.2 + +# our defined job, and its steps jobs: - short-if-irrelevant: + setup: docker: - image: cimg/clojure:1.11.1 steps: @@ -27,497 +27,14 @@ jobs: name: Rename bb binary command: mv /tmp/bb /tmp/bbb - run: - name: Short CI if only irrelevant changes + name: Generate config command: | - /tmp/bbb --config .build/bb.edn short-ci "circleci-agent step halt" - - jvm: - docker: - - image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye - working_directory: ~/repo - environment: - LEIN_ROOT: "true" - BABASHKA_PLATFORM: linux # could be used in jar name - resource_class: large - steps: - - checkout - - run: - name: "Pull Submodules" - command: | - git submodule init - git submodule update - - restore_cache: - keys: - - v1-dependencies-{{ checksum "project.clj" }}-{{ checksum "deps.edn" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - run: - name: Install Clojure - command: | - sudo script/install-clojure - - run: - name: Run JVM tests - command: | - export BABASHKA_FEATURE_JDBC=true - export BABASHKA_FEATURE_POSTGRESQL=true - script/test - script/run_lib_tests - - run: - name: Run as lein command - command: | - .circleci/script/lein - - run: - name: Create uberjar - command: | - mkdir -p /tmp/release - script/uberjar - VERSION=$(cat resources/BABASHKA_VERSION) - jar=target/babashka-$VERSION-standalone.jar - cp $jar /tmp/release - java -jar $jar script/reflection.clj - reflection="babashka-$VERSION-reflection.json" - java -jar "$jar" --config .build/bb.edn --deps-root . release-artifact "$jar" - java -jar "$jar" --config .build/bb.edn --deps-root . release-artifact "$reflection" - - store_artifacts: - path: /tmp/release - destination: release - - save_cache: - paths: - - ~/.m2 - key: v1-dependencies-{{ checksum "project.clj" }}-{{ checksum "deps.edn" }} - linux: - docker: - - image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye - working_directory: ~/repo - environment: - LEIN_ROOT: "true" - GRAALVM_VERSION: "22.1.0" - GRAALVM_HOME: /home/circleci/graalvm-ce-java11-22.1.0 - BABASHKA_PLATFORM: linux # used in release script - BABASHKA_TEST_ENV: native - BABASHKA_XMX: "-J-Xmx6500m" - resource_class: large - steps: - - checkout - - run: - name: "Pull Submodules" - command: | - git submodule init - git submodule update - - restore_cache: - keys: - - linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Clojure - command: | - sudo script/install-clojure - - run: - name: Install native dev tools - command: | - sudo apt-get update - sudo apt-get -y install build-essential zlib1g-dev - - run: - name: Download GraalVM - command: | - script/install-graalvm - - run: - name: Build binary - command: | - script/uberjar - script/compile - no_output_timeout: 30m - - run: - name: Run tests - command: | - script/test - script/run_lib_tests - - run: - name: Release - command: | - .circleci/script/release - - persist_to_workspace: - root: /tmp - paths: - - release - - save_cache: - paths: - - ~/.m2 - - ~/graalvm-ce-java11-22.1.0 - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - store_artifacts: - path: /tmp/release - destination: release - - run: - name: Publish artifact link to Slack - command: | - ./bb .circleci/script/publish_artifact.clj || true - linux-static: - docker: - - image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye - working_directory: ~/repo - environment: - LEIN_ROOT: "true" - GRAALVM_VERSION: "22.1.0" - GRAALVM_HOME: /home/circleci/graalvm-ce-java11-22.1.0 - BABASHKA_PLATFORM: linux # used in release script - BABASHKA_TEST_ENV: native - BABASHKA_STATIC: "true" - BABASHKA_MUSL: "true" - BABASHKA_XMX: "-J-Xmx6500m" - resource_class: large - steps: - - checkout - - attach_workspace: - at: /tmp - - run: - name: "Pull Submodules" - command: | - git submodule init - git submodule update - # - run: - # name: "Short circuit on SNAPSHOT" - # command: | - # VERSION=$(cat resources/BABASHKA_VERSION) - # if [[ "$VERSION" == *-SNAPSHOT ]] - # then - # circleci task halt - # fi - - restore_cache: - keys: - - linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Clojure - command: | - sudo script/install-clojure - - run: - name: Install native dev tools - command: | - sudo apt-get update - sudo apt-get -y install build-essential zlib1g-dev - sudo -E script/setup-musl - - run: - name: Download GraalVM - command: | - script/install-graalvm - - run: - name: Build binary - command: | - script/uberjar - script/compile - no_output_timeout: 30m - - run: - name: Run tests - command: | - script/test - script/run_lib_tests - - run: - name: Release - command: | - .circleci/script/release - - persist_to_workspace: - root: /tmp - paths: - - release - - save_cache: - paths: - - ~/.m2 - - ~/graalvm-ce-java11-22.1.0 - key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - store_artifacts: - path: /tmp/release - destination: release - - run: - name: Publish artifact link to Slack - command: | - ./bb .circleci/script/publish_artifact.clj || true - linux-aarch64: - machine: - image: ubuntu-2004:202111-01 - resource_class: arm.large - working_directory: ~/repo - environment: - LEIN_ROOT: "true" - GRAALVM_VERSION: "22.1.0" - GRAALVM_HOME: /home/circleci/graalvm-ce-java11-22.1.0 - BABASHKA_PLATFORM: linux # used in release script - BABASHKA_ARCH: aarch64 - BABASHKA_TEST_ENV: native - BABASHKA_XMX: "-J-Xmx6500m" - steps: - - checkout - - run: - name: "Pull Submodules" - command: | - git submodule init - git submodule update - - run: - name: Install Clojure - command: | - sudo script/install-clojure - - restore_cache: - keys: - - linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install native dev tools - command: | - sudo apt-get update - sudo apt-get -y install build-essential zlib1g-dev - - run: - name: Download GraalVM - command: | - script/install-graalvm - - run: - name: Build binary - command: | - script/uberjar - script/compile - no_output_timeout: 30m - - run: - name: Run tests - command: | - script/test - script/run_lib_tests - - run: - name: Release - command: | - .circleci/script/release - - persist_to_workspace: - root: /tmp - paths: - - release - - save_cache: - paths: - - ~/.m2 - - ~/graalvm-ce-java11-22.1.0 - key: linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - store_artifacts: - path: /tmp/release - destination: release - - run: - name: Publish artifact link to Slack - command: | - ./bb .circleci/script/publish_artifact.clj || true - linux-aarch64-static: - machine: - image: ubuntu-2004:202111-01 - resource_class: arm.large - working_directory: ~/repo - environment: - LEIN_ROOT: "true" - GRAALVM_VERSION: "22.1.0" - GRAALVM_HOME: /home/circleci/graalvm-ce-java11-22.1.0 - BABASHKA_PLATFORM: linux # used in release script - BABASHKA_ARCH: aarch64 - BABASHKA_TEST_ENV: native - BABASHKA_XMX: "-J-Xmx6500m" - BABASHKA_STATIC: "true" - steps: - - checkout - - run: - name: "Pull Submodules" - command: | - git submodule init - git submodule update - # - run: - # name: "Short circuit on SNAPSHOT" - # command: | - # VERSION=$(cat resources/BABASHKA_VERSION) - # if [[ "$VERSION" == *-SNAPSHOT ]] - # then - # circleci task halt - # fi - - run: - name: Install Clojure - command: | - sudo script/install-clojure - - restore_cache: - keys: - - linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install native dev tools - command: | - sudo apt-get update - sudo apt-get -y install build-essential zlib1g-dev - # sudo -E script/setup-musl - - run: - name: Download GraalVM - command: | - script/install-graalvm - - run: - name: Build binary - command: | - script/uberjar - script/compile - no_output_timeout: 30m - - run: - name: Run tests - command: | - script/test - script/run_lib_tests - - run: - name: Release - command: | - .circleci/script/release - - persist_to_workspace: - root: /tmp - paths: - - release - - save_cache: - paths: - - ~/.m2 - - ~/graalvm-ce-java11-22.1.0 - key: linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - store_artifacts: - path: /tmp/release - destination: release - - run: - name: Publish artifact link to Slack - command: | - ./bb .circleci/script/publish_artifact.clj || true - mac: - macos: - xcode: "12.0.0" - environment: - MACOSX_DEPLOYMENT_TARGET: 10.13 # 10.12 is EOL - GRAALVM_VERSION: "22.1.0" - GRAALVM_HOME: /Users/distiller/graalvm-ce-java11-22.1.0/Contents/Home - BABASHKA_PLATFORM: macos # used in release script - BABASHKA_TEST_ENV: native - BABASHKA_XMX: "-J-Xmx6500m" - resource_class: large - steps: - - checkout - - run: - name: "Pull Submodules" - command: | - git submodule init - git submodule update - - restore_cache: - keys: - - mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Clojure - command: | - script/install-clojure - - run: - name: Install Leiningen - command: | - script/install-leiningen - - run: - name: Download GraalVM - command: | - script/install-graalvm - - run: - name: Build binary - command: | - export PATH=$GRAALVM_HOME/bin:$PATH - script/uberjar - script/compile - no_output_timeout: 30m - - run: - name: Run tests - command: | - export PATH=$GRAALVM_HOME/bin:$PATH - script/test - script/run_lib_tests - - run: - name: Release - command: | - .circleci/script/release - - save_cache: - paths: - - ~/.m2 - - ~/graalvm-ce-java11-22.1.0/Contents/Home - key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - store_artifacts: - path: /tmp/release - destination: release - - run: - name: Publish artifact link to Slack - command: | - ./bb .circleci/script/publish_artifact.clj || true - deploy: - resource_class: large - docker: - - image: circleci/clojure:lein-2.9.8 - working_directory: ~/repo - environment: - LEIN_ROOT: "true" - steps: - - checkout - - run: - name: "Pull Submodules" - command: | - git submodule init - git submodule update - - restore_cache: - keys: - - v1-dependencies-{{ checksum "project.clj" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - run: .circleci/script/deploy - - save_cache: - paths: - - ~/.m2 - key: v1-dependencies-{{ checksum "project.clj" }} - docker: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: - name: "Pull Submodules" - command: | - git submodule init - git submodule update - - setup-docker-buildx - - attach_workspace: - at: /tmp - - run: - name: Build uberjar - command: script/uberjar - - run: - name: Build Docker image - environment: - PLATFORMS: linux/amd64,linux/arm64 - command: java -jar ./target/babashka-$(cat resources/BABASHKA_VERSION)-standalone.jar .circleci/script/docker.clj + /tmp/bbb .circleci/script/short_ci.clj > generated_config.yml + - continuation/continue: + configuration_path: generated_config.yml +# our single workflow, that triggers the setup job defined above workflows: - version: 2 - ci: + setup: jobs: - - short-if-irrelevant - - jvm: - requires: - - short-if-irrelevant - - linux: - requires: - - short-if-irrelevant - - linux-static: - requires: - - short-if-irrelevant - - mac: - requires: - - short-if-irrelevant - - linux-aarch64: - requires: - - short-if-irrelevant - - linux-aarch64-static: - requires: - - short-if-irrelevant - - deploy: - filters: - branches: - only: master - requires: - - jvm - - linux - # - mac # TODO: Enable when not needed from Actions - - docker: - filters: - branches: - only: master - requires: - - linux - - linux-static - - linux-aarch64 + - setup diff --git a/.circleci/script/short_ci.clj b/.circleci/script/short_ci.clj new file mode 100644 index 00000000..3d385efc --- /dev/null +++ b/.circleci/script/short_ci.clj @@ -0,0 +1,250 @@ +(ns short-ci + (:require + [babashka.tasks :as tasks] + [clj-yaml.core :as yaml] + [clojure.string :as str] + [flatland.ordered.map :refer [ordered-map]])) + +(defn run + ([cmd-name cmd] + (run cmd-name cmd nil)) + ([cmd-name cmd no-output-timeout] + (let [base {:run {:name cmd-name + :command cmd}}] + (if no-output-timeout + (assoc-in base [:run :no_output_timeout] no-output-timeout) + base)))) + +(defn gen-steps + [shorted? steps] + (if shorted? + [(run "Shorted" "echo 'Skipping Run'")] + steps)) + +(defn pull-submodules + [] + (run "Pull Submodules" "git submodule init\ngit submodule update")) + +(defn deploy + [shorted?] + (ordered-map + :resource_class "large" + :docker [{:image "circleci/clojure:lein-2.9.8"}] + :working_directory "~/repo" + :environment {:LEIN_ROOT "true"} + :steps (gen-steps + shorted? + [:checkout + (pull-submodules) + {:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}" + "v1-dependencies-"]}} + {:run ".circleci/script/deploy"} + {:save_cache {:paths ["~/.m2"] + :key "v1-dependencies-{{ checksum \"project.clj\" }}"}}]))) + +(defn docker + [shorted?] + (ordered-map + :machine {:image "ubuntu-2004:202111-01"} + :steps + (gen-steps + shorted? + [:checkout + (pull-submodules) + "setup-docker-buildx" + {:attach_workspace {:at "/tmp"}} + (run "Build uberjar" "script/uberjar") + {:run + {:name "Build Docker image" + :environment {:PLATFORMS "linux/amd64,linux/arm64"} + :command + "java -jar ./target/babashka-$(cat resources/BABASHKA_VERSION)-standalone.jar .circleci/script/docker.clj"}}]))) + +(defn jvm + [shorted?] + (ordered-map + :docker [{:image "circleci/clojure:openjdk-11-lein-2.9.8-bullseye"}] + :working_directory "~/repo" + :environment {:LEIN_ROOT "true" + :BABASHKA_PLATFORM "linux"} + :resource_class "large" + :steps + (gen-steps + shorted? + [:checkout + (pull-submodules) + {:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}" + "v1-dependencies-"]}} + (run "Install Clojure" "sudo script/install-clojure") + (run + "Run JVM tests" + "export BABASHKA_FEATURE_JDBC=true +export BABASHKA_FEATURE_POSTGRESQL=true +script/test\nscript/run_lib_tests") + (run "Run as lein command" ".circleci/script/lein") + (run + "Create uberjar" + "mkdir -p /tmp/release +script/uberjar +VERSION=$(cat resources/BABASHKA_VERSION) +jar=target/babashka-$VERSION-standalone.jar +cp $jar /tmp/release +java -jar $jar script/reflection.clj +reflection=\"babashka-$VERSION-reflection.json\" +java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$jar\" +java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$reflection\"") + {:store_artifacts {:path "/tmp/release" + :destination "release"}} + {:save_cache {:paths ["~/.m2"] + :key "v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}"}}]))) + +(defn unix + [shorted? static? musl? arch executor-conf resource-class graalvm-home platform] + (let [env {:LEIN_ROOT "true" + :GRAALVM_VERSION "22.1.0" + :GRAALVM_HOME graalvm-home + :BABASHKA_PLATFORM (if (= "mac" platform) + "macos" + platform) + :BABASHKA_TEST_ENV "native" + :BABASHKA_ARCH arch + :BABASHKA_XMX "-J-Xmx6500m"} + env (if static? + (assoc env :BABASHKA_STATIC "true") + env) + env (if musl? + (assoc env :BABASHKA_MUSL "true") + env) + env (if (= "mac" platform) + (assoc env :MACOSX_DEPLOYMENT_TARGET 10.13) + env) + base-install-cmd "sudo apt-get update\nsudo apt-get -y install build-essential zlib1g-dev" + cache-key (format "%s-%s{{ checksum \"project.clj\" }}-{{ checksum \".circleci/config.yml\" }}" + platform + (if (= "aarch64" arch) + "aarch64-" + ""))] + (merge + executor-conf + (ordered-map + :working_directory "~/repo" + :environment env + :resource_class resource-class + :steps (gen-steps shorted? + (filter some? + [:checkout + {:attach_workspace {:at "/tmp"}} + (run "Pull Submodules" "git submodule init\ngit submodule update") + {:restore_cache + {:keys [cache-key]}} + (run "Install Clojure" "sudo script/install-clojure") + (when (= "mac" platform) + (run "Install Leiningen" "script/install-leiningen")) + (when (not= "mac" platform) + (run "Install native dev tools" + (if (and static? musl? (not= "aarch64" arch)) + (str base-install-cmd "\nsudo -E script/setup-musl") + base-install-cmd))) + (run "Download GraalVM" "script/install-graalvm") + (run "Build binary" "script/uberjar\nscript/compile" "30m") + (run "Run tests" "script/test\nscript/run_lib_tests") + (run "Release" ".circleci/script/release") + {:persist_to_workspace {:root "/tmp" + :paths ["release"]}} + {:save_cache + {:paths ["~/.m2" "~/graalvm-ce-java11-22.1.0"] + :key cache-key}} + {:store_artifacts {:path "/tmp/release" + :destination "release"}} + (run "Publish artifact link to Slack" + "./bb .circleci/script/publish_artifact.clj || true")])))))) + +(defn make-config + [shorted?] + (let [docker-executor-conf {:docker [{:image "circleci/clojure:openjdk-11-lein-2.9.8-bullseye"}]} + machine-executor-conf {:machine {:image "ubuntu-2004:202111-01"}} + mac-executor-conf {:macos {:xcode "12.0.0"}} + linux-graalvm-home "/home/circleci/graalvm-ce-java11-22.1.0" + mac-graalvm-home "/Users/distiller/graalvm-ce-java11-22.1.0/Contents/Home"] + (ordered-map + :version 2.1 + :commands + {:setup-docker-buildx + {:steps + [{:run + {:name "Create multi-platform capabale buildx builder" + :command + "docker run --privileged --rm tonistiigi/binfmt --install all\ndocker buildx create --name ci-builder --use"}}]}} + :jobs (ordered-map + :jvm (jvm shorted?) + :linux (unix shorted? false false "x86_64" docker-executor-conf "large" linux-graalvm-home "linux") + :linux-static + (unix shorted? true true "x86_64" docker-executor-conf "large" linux-graalvm-home "linux") + :linux-aarch64 (unix shorted? + false + false + "aarch64" + machine-executor-conf + "arm.large" + linux-graalvm-home + "linux") + :linux-aarch64-static + (unix shorted? true false "aarch64" machine-executor-conf "arm.large" linux-graalvm-home "linux") + :mac (unix shorted? false false "x86_64" mac-executor-conf "large" mac-graalvm-home "mac") + :deploy (deploy shorted?) + :docker (docker shorted?)) + :workflows (ordered-map + :version 2 + :ci {:jobs ["jvm" + "linux" + "linux-static" + "mac" + "linux-aarch64" + "linux-aarch64-static" + {:deploy {:filters {:branches {:only "master"}} + :requires ["jvm" "linux"]}} + {:docker {:filters {:branches {:only "master"}} + :requires ["linux" "linux-static" "linux-aarch64"]}}]})))) + +(def skip-config + {:skip-if-only [#".*.md$"]}) + +(defn get-changes + [] + (-> (tasks/shell {:out :string} "git diff --name-only HEAD~1") + (:out) + (str/split-lines))) + +(defn irrelevant-change? + [change regexes] + (some? (some #(re-matches % change) regexes))) + +(defn relevant? + [change-set regexes] + (some? (some #(not (irrelevant-change? % regexes)) change-set))) + +(defn main + [] + (let [{:keys [skip-if-only]} skip-config + changed-files (get-changes) + conf (make-config (not (relevant? changed-files skip-if-only)))] + (println (yaml/generate-string conf + :dumper-options + {:flow-style :block})))) + +(when (= *file* (System/getProperty "babashka.file")) + (main)) + +(comment + (main) + (def regexes + [#".*.md$" + #".*.clj$"]) ; ignore clojure files + + (:out (tasks/shell {:out :string} "ls")) + + (irrelevant-change? "src/file.png" regexes) + + (re-matches #".*.clj$" "src/file.clj.dfff") + + (relevant? ["src/file.clj"] regexes))