luciano/babashka
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Optimise executors when shorted (#1301)

Browse source 
* Optimise executors when shorted

* Test short

* Also handle mac

* Test short
This commit is contained in:
Rahuλ Dé 2022-06-21 14:32:23 +01:00 committed by GitHub
parent b5dea3563d
commit 512dbf9ea5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 102 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

190
.circleci/script/short_ci.clj
View file

@ -21,70 +21,83 @@
[(run "Shorted" "echo 'Skipping Run'")] [(run "Shorted" "echo 'Skipping Run'")]
steps)) steps))
(defn gen-job
[shorted? conf]
(if shorted?
(-> conf
(dissoc :machine :macos)
(assoc :resource_class "small" :docker [{:image "ubuntu:latest"}]))
conf))
(defn pull-submodules (defn pull-submodules
[] []
(run "Pull Submodules" "git submodule init\ngit submodule update")) (run "Pull Submodules" "git submodule init\ngit submodule update"))
(defn deploy (defn deploy
[shorted?] [shorted?]
(ordered-map (gen-job shorted?
:resource_class "large" (ordered-map
:docker [{:image "circleci/clojure:lein-2.9.8"}] :resource_class "large"
:working_directory "~/repo" :docker [{:image "circleci/clojure:lein-2.9.8"}]
:environment {:LEIN_ROOT "true"} :working_directory "~/repo"
:steps (gen-steps :environment {:LEIN_ROOT "true"}
shorted? :steps (gen-steps
[:checkout shorted?
(pull-submodules) [:checkout
{:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}" (pull-submodules)
"v1-dependencies-"]}} {:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}"
{:run ".circleci/script/deploy"} "v1-dependencies-"]}}
{:save_cache {:paths ["~/.m2"] {:run ".circleci/script/deploy"}
:key "v1-dependencies-{{ checksum \"project.clj\" }}"}}]))) {:save_cache {:paths ["~/.m2"]
:key "v1-dependencies-{{ checksum \"project.clj\" }}"}}]))))
(defn docker (defn docker
[shorted?] [shorted?]
(ordered-map (gen-job
:machine {:image "ubuntu-2004:202111-01"} shorted?
:steps (ordered-map
(gen-steps :machine {:image "ubuntu-2004:202111-01"}
shorted? :steps
[:checkout (gen-steps
(pull-submodules) shorted?
"setup-docker-buildx" [:checkout
{:attach_workspace {:at "/tmp"}} (pull-submodules)
(run "Build uberjar" "script/uberjar") "setup-docker-buildx"
{:run {:attach_workspace {:at "/tmp"}}
{:name "Build Docker image" (run "Build uberjar" "script/uberjar")
:environment {:PLATFORMS "linux/amd64,linux/arm64"} {:run
:command {:name "Build Docker image"
"java -jar ./target/babashka-$(cat resources/BABASHKA_VERSION)-standalone.jar .circleci/script/docker.clj"}}]))) :environment {:PLATFORMS "linux/amd64,linux/arm64"}
:command
"java -jar ./target/babashka-$(cat resources/BABASHKA_VERSION)-standalone.jar .circleci/script/docker.clj"}}]))))
(defn jvm (defn jvm
[shorted?] [shorted?]
(ordered-map (gen-job
:docker [{:image "circleci/clojure:openjdk-11-lein-2.9.8-bullseye"}] shorted?
:working_directory "~/repo" (ordered-map
:environment {:LEIN_ROOT "true" :docker [{:image "circleci/clojure:openjdk-11-lein-2.9.8-bullseye"}]
:BABASHKA_PLATFORM "linux"} :working_directory "~/repo"
:resource_class "large" :environment {:LEIN_ROOT "true"
:steps :BABASHKA_PLATFORM "linux"}
(gen-steps :resource_class "large"
shorted? :steps
[:checkout (gen-steps
(pull-submodules) shorted?
{:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}" [:checkout
"v1-dependencies-"]}} (pull-submodules)
(run "Install Clojure" "sudo script/install-clojure") {:restore_cache {:keys ["v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}"
(run "v1-dependencies-"]}}
"Run JVM tests" (run "Install Clojure" "sudo script/install-clojure")
"export BABASHKA_FEATURE_JDBC=true (run
"Run JVM tests"
"export BABASHKA_FEATURE_JDBC=true
export BABASHKA_FEATURE_POSTGRESQL=true export BABASHKA_FEATURE_POSTGRESQL=true
script/test\nscript/run_lib_tests") script/test\nscript/run_lib_tests")
(run "Run as lein command" ".circleci/script/lein") (run "Run as lein command" ".circleci/script/lein")
(run (run
"Create uberjar" "Create uberjar"
"mkdir -p /tmp/release "mkdir -p /tmp/release
script/uberjar script/uberjar
VERSION=$(cat resources/BABASHKA_VERSION) VERSION=$(cat resources/BABASHKA_VERSION)
jar=target/babashka-$VERSION-standalone.jar jar=target/babashka-$VERSION-standalone.jar
@ -93,10 +106,10 @@ java -jar $jar script/reflection.clj
reflection=\"babashka-$VERSION-reflection.json\" reflection=\"babashka-$VERSION-reflection.json\"
java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$jar\" java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$jar\"
java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$reflection\"") java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$reflection\"")
{:store_artifacts {:path "/tmp/release" {:store_artifacts {:path "/tmp/release"
:destination "release"}} :destination "release"}}
{:save_cache {:paths ["~/.m2"] {:save_cache {:paths ["~/.m2"]
:key "v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}"}}]))) :key "v1-dependencies-{{ checksum \"project.clj\" }}-{{ checksum \"deps.edn\" }}"}}]))))
(defn unix (defn unix
[shorted? static? musl? arch executor-conf resource-class graalvm-home platform] [shorted? static? musl? arch executor-conf resource-class graalvm-home platform]
@ -126,40 +139,41 @@ java -jar \"$jar\" --config .build/bb.edn --deps-root . release-artifact \"$refl
(if (= "aarch64" arch) (if (= "aarch64" arch)
"aarch64-" "aarch64-"
""))] ""))]
(merge (gen-job shorted?
executor-conf (merge
(ordered-map executor-conf
:working_directory "~/repo" (ordered-map
:environment env :working_directory "~/repo"
:resource_class resource-class :environment env
:steps (gen-steps shorted? :resource_class resource-class
(filter some? :steps (gen-steps shorted?
[:checkout (filter some?
{:attach_workspace {:at "/tmp"}} [:checkout
(run "Pull Submodules" "git submodule init\ngit submodule update") {:attach_workspace {:at "/tmp"}}
{:restore_cache (run "Pull Submodules" "git submodule init\ngit submodule update")
{:keys [cache-key]}} {:restore_cache
(run "Install Clojure" "sudo script/install-clojure") {:keys [cache-key]}}
(when (= "mac" platform) (run "Install Clojure" "sudo script/install-clojure")
(run "Install Leiningen" "script/install-leiningen")) (when (= "mac" platform)
(when (not= "mac" platform) (run "Install Leiningen" "script/install-leiningen"))
(run "Install native dev tools" (when (not= "mac" platform)
(if (and static? musl? (not= "aarch64" arch)) (run "Install native dev tools"
(str base-install-cmd "\nsudo -E script/setup-musl") (if (and static? musl? (not= "aarch64" arch))
base-install-cmd))) (str base-install-cmd "\nsudo -E script/setup-musl")
(run "Download GraalVM" "script/install-graalvm") base-install-cmd)))
(run "Build binary" "script/uberjar\nscript/compile" "30m") (run "Download GraalVM" "script/install-graalvm")
(run "Run tests" "script/test\nscript/run_lib_tests") (run "Build binary" "script/uberjar\nscript/compile" "30m")
(run "Release" ".circleci/script/release") (run "Run tests" "script/test\nscript/run_lib_tests")
{:persist_to_workspace {:root "/tmp" (run "Release" ".circleci/script/release")
:paths ["release"]}} {:persist_to_workspace {:root "/tmp"
{:save_cache :paths ["release"]}}
{:paths ["~/.m2" "~/graalvm-ce-java11-22.1.0"] {:save_cache
:key cache-key}} {:paths ["~/.m2" "~/graalvm-ce-java11-22.1.0"]
{:store_artifacts {:path "/tmp/release" :key cache-key}}
:destination "release"}} {:store_artifacts {:path "/tmp/release"
(run "Publish artifact link to Slack" :destination "release"}}
"./bb .circleci/script/publish_artifact.clj || true")])))))) (run "Publish artifact link to Slack"
"./bb .circleci/script/publish_artifact.clj || true")])))))))
(defn make-config (defn make-config
[shorted?] [shorted?]