luciano/babashka
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feature: declarative pods support in uberjars (#1217)

Browse source 
* Store bb.edn in uberjar & load when run from there

* Sync resources deps.edn w/ root

* Throw error if --config arg doesn't exist

...instead of silently falling back on ./bb.edn

* Fix jar file arg parsing

Moving all arg parsing up above bb.edn ingestion made tasks turn into files to load. But this is a chicken-and-egg problem w/ uberjars b/c the bb.edn is inside the jar file, so we need that first. So I extracted the file parsing out into its own fn and put the full arg parsing back where it was after ingesting bb.edn.

* Fix local pod key in error message

* Ensure resources dir exists when copying bb.edn into it

* Copy bb.edn to temp dir when building uberjar

..and add that dir to the classpath for the jar

* Only put :pods in uberjar bb.edn

* Small readability fix

* Add ret val type hints for classpath/resource

Got a reflection warning on one of my calls

* Add a test for uberjars with bb.edn-declared pods

* Only run uberjar-with-pods-test on amd64 for now
This commit is contained in:
Wes Morgan 2022-04-02 08:51:08 -06:00 committed by GitHub
parent 1cf914ad1c
commit 963bd7957a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 81 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/babashka/impl/classpath.clj
View file

@ -5,7 +5,8 @@
[clojure.java.io :as io]
[clojure.string :as str]
[sci.core :as sci])
(:import [java.util.jar JarFile Manifest]))
(:import [java.util.jar JarFile Manifest]
(java.net URL)))
(set! *warn-on-reflection* true)
@ -106,11 +107,11 @@
[]
(:cp @cp-state))
(defn resource [path]
(when-let [st @cp-state]
(let [loader (:loader st)]
(if (str/starts-with? path "/") nil ;; non-relative paths always return nil
(getResource loader [path] true)))))
(defn resource
(^URL [path] (when-let [st @cp-state] (resource (:loader st) path)))
(^URL [loader path]
(if (str/starts-with? path "/") nil ;; non-relative paths always return nil
(getResource loader [path] true))))
(def cns (sci/create-ns 'babashka.classpath nil))
@ -126,5 +127,5 @@
(def cp "src:feature-xml:feature-core-async:feature-yaml:feature-csv:feature-transit:feature-java-time:feature-java-nio:sci/src:babashka.curl/src:babashka.pods/src:resources:sci/resources:/Users/borkdude/.m2/repository/com/cognitect/transit-java/1.0.343/transit-java-1.0.343.jar:/Users/borkdude/.m2/repository/org/clojure/clojure/1.10.2-alpha1/clojure-1.10.2-alpha1.jar:/Users/borkdude/.m2/repository/commons-codec/commons-codec/1.10/commons-codec-1.10.jar:/Users/borkdude/.m2/repository/org/clojure/tools.analyzer/1.0.0/tools.analyzer-1.0.0.jar:/Users/borkdude/.m2/repository/org/clojure/tools.logging/0.6.0/tools.logging-0.6.0.jar:/Users/borkdude/.m2/repository/org/clojure/core.specs.alpha/0.2.44/core.specs.alpha-0.2.44.jar:/Users/borkdude/.m2/repository/org/clojure/spec.alpha/0.2.187/spec.alpha-0.2.187.jar:/Users/borkdude/.m2/repository/org/clojure/tools.cli/1.0.194/tools.cli-1.0.194.jar:/Users/borkdude/.m2/repository/org/clojure/tools.analyzer.jvm/1.0.0/tools.analyzer.jvm-1.0.0.jar:/Users/borkdude/.m2/repository/borkdude/graal.locking/0.0.2/graal.locking-0.0.2.jar:/Users/borkdude/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.10.2/jackson-dataformat-cbor-2.10.2.jar:/Users/borkdude/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar:/Users/borkdude/.m2/repository/org/flatland/ordered/1.5.9/ordered-1.5.9.jar:/Users/borkdude/.m2/repository/org/postgresql/postgresql/42.2.12/postgresql-42.2.12.jar:/Users/borkdude/.m2/repository/fipp/fipp/0.6.22/fipp-0.6.22.jar:/Users/borkdude/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.10.2/jackson-core-2.10.2.jar:/Users/borkdude/.m2/repository/org/yaml/snakeyaml/1.25/snakeyaml-1.25.jar:/Users/borkdude/.m2/repository/org/ow2/asm/asm/5.2/asm-5.2.jar:/Users/borkdude/.gitlibs/libs/clj-commons/conch/9aa7724e925cb8bf163e0b62486dd420b84e5f0b/src:/Users/borkdude/.m2/repository/org/javassist/javassist/3.18.1-GA/javassist-3.18.1-GA.jar:/Users/borkdude/.m2/repository/seancorfield/next.jdbc/1.0.424/next.jdbc-1.0.424.jar:/Users/borkdude/.m2/repository/org/clojure/data.xml/0.2.0-alpha6/data.xml-0.2.0-alpha6.jar:/Users/borkdude/.m2/repository/org/msgpack/msgpack/0.6.12/msgpack-0.6.12.jar:/Users/borkdude/.m2/repository/borkdude/edamame/0.0.11-alpha.9/edamame-0.0.11-alpha.9.jar:/Users/borkdude/.m2/repository/org/clojure/data.csv/1.0.0/data.csv-1.0.0.jar:/Users/borkdude/.m2/repository/com/cognitect/transit-clj/1.0.324/transit-clj-1.0.324.jar:/Users/borkdude/.m2/repository/clj-commons/clj-yaml/0.7.1/clj-yaml-0.7.1.jar:/Users/borkdude/.m2/repository/org/clojure/core.rrb-vector/0.1.1/core.rrb-vector-0.1.1.jar:/Users/borkdude/.m2/repository/persistent-sorted-set/persistent-sorted-set/0.1.2/persistent-sorted-set-0.1.2.jar:/Users/borkdude/.m2/repository/cheshire/cheshire/5.10.0/cheshire-5.10.0.jar:/Users/borkdude/.m2/repository/tigris/tigris/0.1.2/tigris-0.1.2.jar:/Users/borkdude/.m2/repository/org/clojure/tools.reader/1.3.2/tools.reader-1.3.2.jar:/Users/borkdude/.m2/repository/datascript/datascript/0.18.11/datascript-0.18.11.jar:/Users/borkdude/.m2/repository/org/hsqldb/hsqldb/2.4.0/hsqldb-2.4.0.jar:/Users/borkdude/.m2/repository/org/clojure/core.memoize/0.8.2/core.memoize-0.8.2.jar:/Users/borkdude/.m2/repository/org/clojure/data.priority-map/0.0.7/data.priority-map-0.0.7.jar:/Users/borkdude/.m2/repository/org/clojure/java.data/1.0.64/java.data-1.0.64.jar:/Users/borkdude/.m2/repository/borkdude/sci.impl.reflector/0.0.1/sci.impl.reflector-0.0.1.jar:/Users/borkdude/.m2/repository/nrepl/bencode/1.1.0/bencode-1.1.0.jar:/Users/borkdude/.m2/repository/org/clojure/core.cache/0.8.2/core.cache-0.8.2.jar:/Users/borkdude/.m2/repository/org/clojure/core.async/1.1.587/core.async-1.1.587.jar:/Users/borkdude/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-smile/2.10.2/jackson-dataformat-smile-2.10.2.jar:/Users/borkdude/.m2/repository/org/clojure/data.codec/0.1.0/data.codec-0.1.0.jar:/Users/borkdude/.m2/repository/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar")
(def l (loader cp))
(source-for-namespace l 'babashka.impl.cheshire nil)
(time (:file (source-for-namespace l 'cheshire.core nil))) ;; 20ms -> 2.25ms
)
(time (:file (source-for-namespace l 'cheshire.core nil)))) ;; 20ms -> 2.25ms

2
src/babashka/impl/pods.clj
View file

@ -28,7 +28,7 @@
(throw (IllegalArgumentException.
(str (-> coord keys first)
" is not a supported pod coordinate type. "
"Use :version for registry-hosted pods or :local/root "
"Use :version for registry-hosted pods or :path "
"for pods on your local filesystem."))))))
{} pods-map))

60
src/babashka/main.clj
View file

@ -663,6 +663,15 @@ Use bb run --help to show this help output.
[options opts-map])
[options opts-map])))
(defn parse-file-opt
[options opts-map]
(let [opt (first options)
opts-key (if (str/ends-with? opt ".jar")
:jar :file)]
(assoc opts-map
opts-key opt
:command-line-args (next options))))
(defn parse-opts
([options] (parse-opts options nil))
([options opts-map]
@ -672,19 +681,18 @@ Use bb run --help to show this help output.
;; FILE > TASK > SUBCOMMAND
(cond
(.isFile (io/file opt))
(if (str/ends-with? opt ".jar")
(assoc opts-map
:jar opt
:command-line-args (next options))
(assoc opts-map
:file opt
:command-line-args (next options)))
(if (or (:file opts-map) (:jar opts-map))
opts-map ; we've already parsed the file opt
(parse-file-opt options opts-map))
(contains? tasks opt)
(assoc opts-map
:run opt
:command-line-args (next options))
(command? opt)
(recur (cons (str "--" opt) (next options)) opts-map)
:else
(parse-args options opts-map))))))
@ -920,11 +928,19 @@ Use bb run --help to show this help output.
(spit uberscript-out expression :append true)))
(when uberjar
(if-let [cp (cp/get-classpath)]
(uberjar/run {:dest uberjar
:jar :uber
:classpath cp
:main-class main
:verbose debug})
(let [uber-params {:dest uberjar
:jar :uber
:classpath cp
:main-class main
:verbose debug}]
(if-let [bb-edn-pods (:pods @common/bb-edn)]
(fs/with-temp-dir [bb-edn-dir {}]
(let [bb-edn-resource (fs/file bb-edn-dir "bb.edn")]
(->> {:pods bb-edn-pods} pr-str (spit bb-edn-resource))
(let [cp-with-bb-edn (str cp cp/path-sep bb-edn-dir)]
(uberjar/run (assoc uber-params
:classpath cp-with-bb-edn)))))
(uberjar/run uber-params)))
(throw (Exception. "The uberjar task needs a classpath."))))
exit-code))))
@ -939,17 +955,21 @@ Use bb run --help to show this help output.
(defn main [& args]
(let [[args global-opts] (parse-global-opts args)
{:keys [:jar] :as file-opt} (when (some-> args first io/file .isFile)
(parse-file-opt args global-opts))
config (:config global-opts)
bb-edn-file (or config
"bb.edn")
bb-edn (when (fs/exists? bb-edn-file)
(System/setProperty "babashka.config"
(.getAbsolutePath (io/file bb-edn-file)))
abs-path #(-> % io/file .getAbsolutePath)
bb-edn-file (cond
config (when (fs/exists? config) (abs-path config))
jar (some-> jar cp/loader (cp/resource "bb.edn") .toString)
:else (when (fs/exists? "bb.edn") (abs-path "bb.edn")))
bb-edn (when bb-edn-file
(System/setProperty "babashka.config" bb-edn-file)
(let [raw-string (slurp bb-edn-file)
edn (edn/read-string raw-string)
edn (assoc edn
:raw raw-string
:file bb-edn-file)
:raw raw-string
:file bb-edn-file)
edn (if-let [deps-root (or (:deps-root global-opts)
(some-> config fs/parent))]
(assoc edn :deps-root deps-root)
@ -961,7 +981,7 @@ Use bb run --help to show this help output.
(binding [*out* *err*]
(println (str "WARNING: this project requires babashka "
min-bb-version " or newer, but you have: " version)))))
(exec (parse-opts args global-opts))))
(exec (parse-opts args (merge global-opts file-opt)))))
(def musl?
"Captured at compile time, to know if we are running inside a

37
test/babashka/uberjar_test.clj
View file

@ -3,14 +3,21 @@
[babashka.main :as main]
[babashka.test-utils :as tu]
[clojure.string :as str]
[clojure.test :as t :refer [deftest is testing]]))
[clojure.test :as t :refer [deftest is testing]]
[babashka.fs :as fs]
[clojure.edn :as edn])
(:import (java.util.jar JarFile)
(java.io File InputStreamReader PushbackReader)))
(defn jar-entries [jar]
(with-open [jar-file (JarFile. jar)]
(doall (enumeration-seq (.entries jar-file)))))
(defn count-entries [jar]
(with-open [jar-file (java.util.jar.JarFile. jar)]
(count (map #_prn
identity
(enumeration-seq
(.entries jar-file))))))
(-> jar jar-entries count))
(defn get-entry [^File jar entry-name]
(-> jar JarFile. (.getEntry entry-name)))
(deftest uberjar-test
(testing "uberjar with --main"
@ -54,6 +61,24 @@
;; Only a manifest entry is added
(is (< (count-entries path) 3)))))
(deftest uberjar-with-pods-test
(testing "jar contains bb.edn w/ only :pods when bb.edn has :pods"
(when (= "amd64" (System/getProperty "os.arch")) ; TODO: Build bootleg for aarch64 too or use a different pod
(let [tmp-file (java.io.File/createTempFile "uber" ".jar")
path (.getPath tmp-file)]
(.deleteOnExit tmp-file)
(let [config {:paths ["test-resources/babashka/uberjar/src"]
:deps '{local/deps {:local/root "test-resources/babashka/uberjar"}}
:pods '{retrogradeorbit/bootleg {:version "0.1.9"}
pod/test-pod {:path "test-resources/pod"}}}]
(tu/with-config config
(tu/bb nil "uberjar" path "-m" "my.main-main")
(let [bb-edn-entry (get-entry tmp-file "bb.edn")
bb-edn (-> path JarFile. (.getInputStream bb-edn-entry)
InputStreamReader. PushbackReader. edn/read)]
(is (= #{:pods} (-> bb-edn keys set)))
(is (= (:pods config) (:pods bb-edn))))))))))
(deftest throw-on-empty-classpath
;; this test fails the windows native test in CI
(when-not main/windows?