From 963bd7957a77f42bf1de1625e5c3bb19ace72107 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Sat, 2 Apr 2022 08:51:08 -0600 Subject: [PATCH] Feature: declarative pods support in uberjars (#1217) * Store bb.edn in uberjar & load when run from there * Sync resources deps.edn w/ root * Throw error if --config arg doesn't exist ...instead of silently falling back on ./bb.edn * Fix jar file arg parsing Moving all arg parsing up above bb.edn ingestion made tasks turn into files to load. But this is a chicken-and-egg problem w/ uberjars b/c the bb.edn is inside the jar file, so we need that first. So I extracted the file parsing out into its own fn and put the full arg parsing back where it was after ingesting bb.edn. * Fix local pod key in error message * Ensure resources dir exists when copying bb.edn into it * Copy bb.edn to temp dir when building uberjar ..and add that dir to the classpath for the jar * Only put :pods in uberjar bb.edn * Small readability fix * Add ret val type hints for classpath/resource Got a reflection warning on one of my calls * Add a test for uberjars with bb.edn-declared pods * Only run uberjar-with-pods-test on amd64 for now --- src/babashka/impl/classpath.clj | 17 +++++----- src/babashka/impl/pods.clj | 2 +- src/babashka/main.clj | 60 ++++++++++++++++++++++----------- test/babashka/uberjar_test.clj | 37 ++++++++++++++++---- 4 files changed, 81 insertions(+), 35 deletions(-) diff --git a/src/babashka/impl/classpath.clj b/src/babashka/impl/classpath.clj index cd223e1f..1506b80e 100644 --- a/src/babashka/impl/classpath.clj +++ b/src/babashka/impl/classpath.clj @@ -5,7 +5,8 @@ [clojure.java.io :as io] [clojure.string :as str] [sci.core :as sci]) - (:import [java.util.jar JarFile Manifest])) + (:import [java.util.jar JarFile Manifest] + (java.net URL))) (set! *warn-on-reflection* true) @@ -106,11 +107,11 @@ [] (:cp @cp-state)) -(defn resource [path] - (when-let [st @cp-state] - (let [loader (:loader st)] - (if (str/starts-with? path "/") nil ;; non-relative paths always return nil - (getResource loader [path] true))))) +(defn resource + (^URL [path] (when-let [st @cp-state] (resource (:loader st) path))) + (^URL [loader path] + (if (str/starts-with? path "/") nil ;; non-relative paths always return nil + (getResource loader [path] true)))) (def cns (sci/create-ns 'babashka.classpath nil)) @@ -126,5 +127,5 @@ (def cp "src:feature-xml:feature-core-async:feature-yaml:feature-csv:feature-transit:feature-java-time:feature-java-nio:sci/src:babashka.curl/src:babashka.pods/src:resources:sci/resources:/Users/borkdude/.m2/repository/com/cognitect/transit-java/1.0.343/transit-java-1.0.343.jar:/Users/borkdude/.m2/repository/org/clojure/clojure/1.10.2-alpha1/clojure-1.10.2-alpha1.jar:/Users/borkdude/.m2/repository/commons-codec/commons-codec/1.10/commons-codec-1.10.jar:/Users/borkdude/.m2/repository/org/clojure/tools.analyzer/1.0.0/tools.analyzer-1.0.0.jar:/Users/borkdude/.m2/repository/org/clojure/tools.logging/0.6.0/tools.logging-0.6.0.jar:/Users/borkdude/.m2/repository/org/clojure/core.specs.alpha/0.2.44/core.specs.alpha-0.2.44.jar:/Users/borkdude/.m2/repository/org/clojure/spec.alpha/0.2.187/spec.alpha-0.2.187.jar:/Users/borkdude/.m2/repository/org/clojure/tools.cli/1.0.194/tools.cli-1.0.194.jar:/Users/borkdude/.m2/repository/org/clojure/tools.analyzer.jvm/1.0.0/tools.analyzer.jvm-1.0.0.jar:/Users/borkdude/.m2/repository/borkdude/graal.locking/0.0.2/graal.locking-0.0.2.jar:/Users/borkdude/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.10.2/jackson-dataformat-cbor-2.10.2.jar:/Users/borkdude/.m2/repository/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar:/Users/borkdude/.m2/repository/org/flatland/ordered/1.5.9/ordered-1.5.9.jar:/Users/borkdude/.m2/repository/org/postgresql/postgresql/42.2.12/postgresql-42.2.12.jar:/Users/borkdude/.m2/repository/fipp/fipp/0.6.22/fipp-0.6.22.jar:/Users/borkdude/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.10.2/jackson-core-2.10.2.jar:/Users/borkdude/.m2/repository/org/yaml/snakeyaml/1.25/snakeyaml-1.25.jar:/Users/borkdude/.m2/repository/org/ow2/asm/asm/5.2/asm-5.2.jar:/Users/borkdude/.gitlibs/libs/clj-commons/conch/9aa7724e925cb8bf163e0b62486dd420b84e5f0b/src:/Users/borkdude/.m2/repository/org/javassist/javassist/3.18.1-GA/javassist-3.18.1-GA.jar:/Users/borkdude/.m2/repository/seancorfield/next.jdbc/1.0.424/next.jdbc-1.0.424.jar:/Users/borkdude/.m2/repository/org/clojure/data.xml/0.2.0-alpha6/data.xml-0.2.0-alpha6.jar:/Users/borkdude/.m2/repository/org/msgpack/msgpack/0.6.12/msgpack-0.6.12.jar:/Users/borkdude/.m2/repository/borkdude/edamame/0.0.11-alpha.9/edamame-0.0.11-alpha.9.jar:/Users/borkdude/.m2/repository/org/clojure/data.csv/1.0.0/data.csv-1.0.0.jar:/Users/borkdude/.m2/repository/com/cognitect/transit-clj/1.0.324/transit-clj-1.0.324.jar:/Users/borkdude/.m2/repository/clj-commons/clj-yaml/0.7.1/clj-yaml-0.7.1.jar:/Users/borkdude/.m2/repository/org/clojure/core.rrb-vector/0.1.1/core.rrb-vector-0.1.1.jar:/Users/borkdude/.m2/repository/persistent-sorted-set/persistent-sorted-set/0.1.2/persistent-sorted-set-0.1.2.jar:/Users/borkdude/.m2/repository/cheshire/cheshire/5.10.0/cheshire-5.10.0.jar:/Users/borkdude/.m2/repository/tigris/tigris/0.1.2/tigris-0.1.2.jar:/Users/borkdude/.m2/repository/org/clojure/tools.reader/1.3.2/tools.reader-1.3.2.jar:/Users/borkdude/.m2/repository/datascript/datascript/0.18.11/datascript-0.18.11.jar:/Users/borkdude/.m2/repository/org/hsqldb/hsqldb/2.4.0/hsqldb-2.4.0.jar:/Users/borkdude/.m2/repository/org/clojure/core.memoize/0.8.2/core.memoize-0.8.2.jar:/Users/borkdude/.m2/repository/org/clojure/data.priority-map/0.0.7/data.priority-map-0.0.7.jar:/Users/borkdude/.m2/repository/org/clojure/java.data/1.0.64/java.data-1.0.64.jar:/Users/borkdude/.m2/repository/borkdude/sci.impl.reflector/0.0.1/sci.impl.reflector-0.0.1.jar:/Users/borkdude/.m2/repository/nrepl/bencode/1.1.0/bencode-1.1.0.jar:/Users/borkdude/.m2/repository/org/clojure/core.cache/0.8.2/core.cache-0.8.2.jar:/Users/borkdude/.m2/repository/org/clojure/core.async/1.1.587/core.async-1.1.587.jar:/Users/borkdude/.m2/repository/com/fasterxml/jackson/dataformat/jackson-dataformat-smile/2.10.2/jackson-dataformat-smile-2.10.2.jar:/Users/borkdude/.m2/repository/org/clojure/data.codec/0.1.0/data.codec-0.1.0.jar:/Users/borkdude/.m2/repository/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar") (def l (loader cp)) (source-for-namespace l 'babashka.impl.cheshire nil) - (time (:file (source-for-namespace l 'cheshire.core nil))) ;; 20ms -> 2.25ms - ) + (time (:file (source-for-namespace l 'cheshire.core nil)))) ;; 20ms -> 2.25ms + diff --git a/src/babashka/impl/pods.clj b/src/babashka/impl/pods.clj index 37923471..fb07c087 100644 --- a/src/babashka/impl/pods.clj +++ b/src/babashka/impl/pods.clj @@ -28,7 +28,7 @@ (throw (IllegalArgumentException. (str (-> coord keys first) " is not a supported pod coordinate type. " - "Use :version for registry-hosted pods or :local/root " + "Use :version for registry-hosted pods or :path " "for pods on your local filesystem.")))))) {} pods-map)) diff --git a/src/babashka/main.clj b/src/babashka/main.clj index 9aa73460..b957d0c3 100644 --- a/src/babashka/main.clj +++ b/src/babashka/main.clj @@ -663,6 +663,15 @@ Use bb run --help to show this help output. [options opts-map]) [options opts-map]))) +(defn parse-file-opt + [options opts-map] + (let [opt (first options) + opts-key (if (str/ends-with? opt ".jar") + :jar :file)] + (assoc opts-map + opts-key opt + :command-line-args (next options)))) + (defn parse-opts ([options] (parse-opts options nil)) ([options opts-map] @@ -672,19 +681,18 @@ Use bb run --help to show this help output. ;; FILE > TASK > SUBCOMMAND (cond (.isFile (io/file opt)) - (if (str/ends-with? opt ".jar") - (assoc opts-map - :jar opt - :command-line-args (next options)) - (assoc opts-map - :file opt - :command-line-args (next options))) + (if (or (:file opts-map) (:jar opts-map)) + opts-map ; we've already parsed the file opt + (parse-file-opt options opts-map)) + (contains? tasks opt) (assoc opts-map :run opt :command-line-args (next options)) + (command? opt) (recur (cons (str "--" opt) (next options)) opts-map) + :else (parse-args options opts-map)))))) @@ -920,11 +928,19 @@ Use bb run --help to show this help output. (spit uberscript-out expression :append true))) (when uberjar (if-let [cp (cp/get-classpath)] - (uberjar/run {:dest uberjar - :jar :uber - :classpath cp - :main-class main - :verbose debug}) + (let [uber-params {:dest uberjar + :jar :uber + :classpath cp + :main-class main + :verbose debug}] + (if-let [bb-edn-pods (:pods @common/bb-edn)] + (fs/with-temp-dir [bb-edn-dir {}] + (let [bb-edn-resource (fs/file bb-edn-dir "bb.edn")] + (->> {:pods bb-edn-pods} pr-str (spit bb-edn-resource)) + (let [cp-with-bb-edn (str cp cp/path-sep bb-edn-dir)] + (uberjar/run (assoc uber-params + :classpath cp-with-bb-edn))))) + (uberjar/run uber-params))) (throw (Exception. "The uberjar task needs a classpath.")))) exit-code)))) @@ -939,17 +955,21 @@ Use bb run --help to show this help output. (defn main [& args] (let [[args global-opts] (parse-global-opts args) + {:keys [:jar] :as file-opt} (when (some-> args first io/file .isFile) + (parse-file-opt args global-opts)) config (:config global-opts) - bb-edn-file (or config - "bb.edn") - bb-edn (when (fs/exists? bb-edn-file) - (System/setProperty "babashka.config" - (.getAbsolutePath (io/file bb-edn-file))) + abs-path #(-> % io/file .getAbsolutePath) + bb-edn-file (cond + config (when (fs/exists? config) (abs-path config)) + jar (some-> jar cp/loader (cp/resource "bb.edn") .toString) + :else (when (fs/exists? "bb.edn") (abs-path "bb.edn"))) + bb-edn (when bb-edn-file + (System/setProperty "babashka.config" bb-edn-file) (let [raw-string (slurp bb-edn-file) edn (edn/read-string raw-string) edn (assoc edn - :raw raw-string - :file bb-edn-file) + :raw raw-string + :file bb-edn-file) edn (if-let [deps-root (or (:deps-root global-opts) (some-> config fs/parent))] (assoc edn :deps-root deps-root) @@ -961,7 +981,7 @@ Use bb run --help to show this help output. (binding [*out* *err*] (println (str "WARNING: this project requires babashka " min-bb-version " or newer, but you have: " version))))) - (exec (parse-opts args global-opts)))) + (exec (parse-opts args (merge global-opts file-opt))))) (def musl? "Captured at compile time, to know if we are running inside a diff --git a/test/babashka/uberjar_test.clj b/test/babashka/uberjar_test.clj index 66d94a13..fe2ed56a 100644 --- a/test/babashka/uberjar_test.clj +++ b/test/babashka/uberjar_test.clj @@ -3,14 +3,21 @@ [babashka.main :as main] [babashka.test-utils :as tu] [clojure.string :as str] - [clojure.test :as t :refer [deftest is testing]])) + [clojure.test :as t :refer [deftest is testing]] + [babashka.fs :as fs] + [clojure.edn :as edn]) + (:import (java.util.jar JarFile) + (java.io File InputStreamReader PushbackReader))) + +(defn jar-entries [jar] + (with-open [jar-file (JarFile. jar)] + (doall (enumeration-seq (.entries jar-file))))) (defn count-entries [jar] - (with-open [jar-file (java.util.jar.JarFile. jar)] - (count (map #_prn - identity - (enumeration-seq - (.entries jar-file)))))) + (-> jar jar-entries count)) + +(defn get-entry [^File jar entry-name] + (-> jar JarFile. (.getEntry entry-name))) (deftest uberjar-test (testing "uberjar with --main" @@ -54,6 +61,24 @@ ;; Only a manifest entry is added (is (< (count-entries path) 3))))) +(deftest uberjar-with-pods-test + (testing "jar contains bb.edn w/ only :pods when bb.edn has :pods" + (when (= "amd64" (System/getProperty "os.arch")) ; TODO: Build bootleg for aarch64 too or use a different pod + (let [tmp-file (java.io.File/createTempFile "uber" ".jar") + path (.getPath tmp-file)] + (.deleteOnExit tmp-file) + (let [config {:paths ["test-resources/babashka/uberjar/src"] + :deps '{local/deps {:local/root "test-resources/babashka/uberjar"}} + :pods '{retrogradeorbit/bootleg {:version "0.1.9"} + pod/test-pod {:path "test-resources/pod"}}}] + (tu/with-config config + (tu/bb nil "uberjar" path "-m" "my.main-main") + (let [bb-edn-entry (get-entry tmp-file "bb.edn") + bb-edn (-> path JarFile. (.getInputStream bb-edn-entry) + InputStreamReader. PushbackReader. edn/read)] + (is (= #{:pods} (-> bb-edn keys set))) + (is (= (:pods config) (:pods bb-edn)))))))))) + (deftest throw-on-empty-classpath ;; this test fails the windows native test in CI (when-not main/windows?