From 512dbf9ea54f0ca8a49e247ee78f7fda7f070d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rahu=CE=BB=20D=C3=A9?= Date: Tue, 21 Jun 2022 14:32:23 +0100 Subject: [PATCH] Optimise executors when shorted (#1301) * Optimise executors when shorted * Test short * Also handle mac * Test short --- .circleci/script/short_ci.clj | 190 ++++++++++++++++++---------------- 1 file changed, 102 insertions(+), 88 deletions(-) diff --git a/.circleci/script/short_ci.clj b/.circleci/script/short_ci.clj index ffbf3c47..a91ff1f8 100644 --- a/.circleci/script/short_ci.clj +++ b/.circleci/script/short_ci.clj @@ -21,70 +21,83 @@ [(run "Shorted" "echo 'Skipping Run'")] steps)) +(defn gen-job + [shorted? conf] + (if shorted? + (-> conf + (dissoc :machine :macos) + (assoc :resource_class "small" :docker [{:image "ubuntu:latest"}])) + conf)) + (defn pull-submodules [] (run "Pull Submodules" "git submodule init\ngit submodule update")) (defn deploy [shorted?] - (ordered-map - :resource_class "large" - :docker [{:image "circleci/clojure:lein-2.9.8"}] - :working_directory "~/repo" - :environment {:LEIN_ROOT "true"} - :steps (gen-steps - shorted? - [:checkout - (pull-submodules) - {:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}" - "v1-dependencies-"]}} - {:run ".circleci/script/deploy"} - {:save_cache {:paths ["~/.m2"] - :key "v1-dependencies-{{ checksum \"project.clj\" }}"}}]))) + (gen-job shorted? + (ordered-map + :resource_class "large" + :docker [{:image "circleci/clojure:lein-2.9.8"}] + :working_directory "~/repo" + :environment {:LEIN_ROOT "true"} + :steps (gen-steps + shorted? + [:checkout + (pull-submodules) + {:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}" + "v1-dependencies-"]}} + {:run ".circleci/script/deploy"} + {:save_cache {:paths ["~/.m2"] + :key "v1-dependencies-{{ checksum \"project.clj\" }}"}}])))) (defn docker [shorted?] - (ordered-map - :machine {:image "ubuntu-2004:202111-01"} - :steps - (gen-steps - shorted? - [:checkout - (pull-submodules) - "setup-docker-buildx" - {:attach_workspace {:at "/tmp"}} - (run "Build uberjar" "script/uberjar") - {:run - {:name "Build Docker image" - :environment {:PLATFORMS "linux/amd64,linux/arm64"} - :command - "java -jar ./target/babashka-$(cat resources/BABASHKA_VERSION)-standalone.jar .circleci/script/docker.clj"}}]))) + (gen-job + shorted? + (ordered-map + :machine {:image "ubuntu-2004:202111-01"} + :steps + (gen-steps + shorted? + [:checkout + (pull-submodules) + "setup-docker-buildx" + {:attach_workspace {:at "/tmp"}} + (run "Build uberjar" "script/uberjar") + {:run + {:name "Build Docker image" + :environment {:PLATFORMS "linux/amd64,linux/arm64"} + :command + "java -jar ./target/babashka-$(cat resources/BABASHKA_VERSION)-standalone.jar .circleci/script/docker.clj"}}])))) (defn jvm [shorted?] - (ordered-map - :docker [{:image "circleci/clojure:openjdk-11-lein-2.9.8-bullseye"}] - :working_directory "~/repo" - :environment {:LEIN_ROOT "true" - :BABASHKA_PLATFORM "linux"} - :resource_class "large" - :steps - (gen-steps - shorted? - [:checkout - (pull-submodules) - {:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}" - "v1-dependencies-"]}} - (run "Install Clojure" "sudo script/install-clojure") - (run - "Run JVM tests" - "export BABASHKA_FEATURE_JDBC=true + (gen-job + shorted? + (ordered-map + :docker [{:image "circleci/clojure:openjdk-11-lein-2.9.8-bullseye"}] + :working_directory "~/repo" + :environment {:LEIN_ROOT "true" + :BABASHKA_PLATFORM "linux"} + :resource_class "large" + :steps + (gen-steps + shorted? + [:checkout + (pull-submodules) + {:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}" + "v1-dependencies-"]}} + (run "Install Clojure" "sudo script/install-clojure") + (run + "Run JVM tests" + "export BABASHKA_FEATURE_JDBC=true export BABASHKA_FEATURE_POSTGRESQL=true script/test\nscript/run_lib_tests") - (run "Run as lein command" ".circleci/script/lein") - (run - "Create uberjar" - "mkdir -p /tmp/release + (run "Run as lein command" ".circleci/script/lein") + (run + "Create uberjar" + "mkdir -p /tmp/release script/uberjar VERSION=$(cat resources/BABASHKA_VERSION) jar=target/babashka-$VERSION-standalone.jar @@ -93,10 +106,10 @@ java -jar $jar script/reflection.clj reflection=\"babashka-$VERSION-reflection.json\" java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$jar\" java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$reflection\"") - {:store_artifacts {:path "/tmp/release" - :destination "release"}} - {:save_cache {:paths ["~/.m2"] - :key "v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}"}}]))) + {:store_artifacts {:path "/tmp/release" + :destination "release"}} + {:save_cache {:paths ["~/.m2"] + :key "v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}"}}])))) (defn unix [shorted? static? musl? arch executor-conf resource-class graalvm-home platform] @@ -126,40 +139,41 @@ java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$refl (if (= "aarch64" arch) "aarch64-" ""))] - (merge - executor-conf - (ordered-map - :working_directory "~/repo" - :environment env - :resource_class resource-class - :steps (gen-steps shorted? - (filter some? - [:checkout - {:attach_workspace {:at "/tmp"}} - (run "Pull Submodules" "git submodule init\ngit submodule update") - {:restore_cache - {:keys [cache-key]}} - (run "Install Clojure" "sudo script/install-clojure") - (when (= "mac" platform) - (run "Install Leiningen" "script/install-leiningen")) - (when (not= "mac" platform) - (run "Install native dev tools" - (if (and static? musl? (not= "aarch64" arch)) - (str base-install-cmd "\nsudo -E script/setup-musl") - base-install-cmd))) - (run "Download GraalVM" "script/install-graalvm") - (run "Build binary" "script/uberjar\nscript/compile" "30m") - (run "Run tests" "script/test\nscript/run_lib_tests") - (run "Release" ".circleci/script/release") - {:persist_to_workspace {:root "/tmp" - :paths ["release"]}} - {:save_cache - {:paths ["~/.m2" "~/graalvm-ce-java11-22.1.0"] - :key cache-key}} - {:store_artifacts {:path "/tmp/release" - :destination "release"}} - (run "Publish artifact link to Slack" - "./bb .circleci/script/publish_artifact.clj || true")])))))) + (gen-job shorted? + (merge + executor-conf + (ordered-map + :working_directory "~/repo" + :environment env + :resource_class resource-class + :steps (gen-steps shorted? + (filter some? + [:checkout + {:attach_workspace {:at "/tmp"}} + (run "Pull Submodules" "git submodule init\ngit submodule update") + {:restore_cache + {:keys [cache-key]}} + (run "Install Clojure" "sudo script/install-clojure") + (when (= "mac" platform) + (run "Install Leiningen" "script/install-leiningen")) + (when (not= "mac" platform) + (run "Install native dev tools" + (if (and static? musl? (not= "aarch64" arch)) + (str base-install-cmd "\nsudo -E script/setup-musl") + base-install-cmd))) + (run "Download GraalVM" "script/install-graalvm") + (run "Build binary" "script/uberjar\nscript/compile" "30m") + (run "Run tests" "script/test\nscript/run_lib_tests") + (run "Release" ".circleci/script/release") + {:persist_to_workspace {:root "/tmp" + :paths ["release"]}} + {:save_cache + {:paths ["~/.m2" "~/graalvm-ce-java11-22.1.0"] + :key cache-key}} + {:store_artifacts {:path "/tmp/release" + :destination "release"}} + (run "Publish artifact link to Slack" + "./bb .circleci/script/publish_artifact.clj || true")]))))))) (defn make-config [shorted?]