[#536] Support uberjars

.gitmodules

@@ -11,3 +11,6 @@
 [submodule "babashka.nrepl"]
 	path = babashka.nrepl
 	url = https://github.com/babashka/babashka.nrepl
+[submodule "depstar"]
+	path = depstar
+	url = https://github.com/babashka/depstar

README.md

@@ -562,6 +562,85 @@ namespace which allows dynamically adding to the classpath.
 
 See [deps.clj](doc/deps.clj.md) for a babashka script that replaces the `clojure` bash script.
 
+## Uberscript
+
+The `--uberscript` option collects the expressions in
+`BABASHKA_PRELOADS`, the command line expression or file, the main entrypoint
+and all required namespaces from the classpath into a single file. This can be
+convenient for debugging and deployment.
+
+Given the `deps.edn` from above:
+
+``` clojure
+$ deps.clj -A:my-script -Scommand "bb -cp {{classpath}} {{main-opts}} --uberscript my-script.clj"
+
+$ cat my-script.clj
+(ns my-gist-script)
+(defn -main [& args]
+  (println "Hello from gist script!"))
+(ns user (:require [my-gist-script]))
+(apply my-gist-script/-main *command-line-args*)
+
+$ bb my-script.clj
+Hello from gist script!
+```
+
+Caveats:
+
+- *Dynamic requires*. Building uberscripts works by running top-level `ns` and
+`require` forms. The rest of the code is not evaluated. Code that relies on
+dynamic requires may not work in an uberscript.
+- *Resources*. The usage of `io/resource` assumes a classpath, so when this is
+  used in your uberscript, you still have to set a classpath and bring the
+  resources along.
+
+If any of the above is problematic for your project, using an uberjar (see
+below) is a good alternative.
+
+## Uberjar
+
+Babashka can create uberjars from a given classpath and optionally a main
+method:
+
+``` shell
+$ bb -cp $(clojure -Spath) -m my-gist-script --uberjar my-project.jar
+$ bb my-project.jar
+Hello from gist script!
+```
+
+When producing a classpath using the `clojure` or `deps.clj` tool, Clojure
+itself, spec and the core specs will be on the classpath and will therefore be
+included in your uberjar, which makes it bigger than necessary:
+
+``` shell
+$ ls -lh my-project.jar
+-rw-r--r--  1 borkdude  staff   4.5M Aug 19 14:45 my-project.jar
+```
+
+To exclude these dependencies, you can use the following `:classpath-overrides`
+in your `deps.edn`:
+
+``` clojure
+{:deps
+ {my_gist_script
+  {:git/url "https://gist.github.com/borkdude/263b150607f3ce03630e114611a4ef42"
+   :sha "cfc761d06dfb30bb77166b45d439fe8fe54a31b8"}}
+ :aliases {:my-script {:main-opts ["-m" "my-gist-script"]}
+           :remove-clojure {:classpath-overrides {org.clojure/clojure nil
+                                                  org.clojure/spec.alpha nil
+                                                  org.clojure/core.specs.alpha nil}}}}
+
+```
+
+``` shell
+$ bb -cp $(clojure -A:remove-clojure -Spath) -m my-gist-script --uberjar my-bb-project.jar
+$ bb my-project.jar
+Hello from gist script!
+$ ls -lat *.jar
+-rw-r--r--  1 borkdude  staff  4682045 Aug 19 14:55 my-project.jar
+-rw-r--r--  1 borkdude  staff     7880 Aug 19 14:55 my-bb-project.jar
+```
+
 ## System properties
 
 Babashka sets the following system properties:
@@ -593,33 +672,6 @@ $ bb "(set! *data-readers* {'t/tag inc}) #t/tag 1"
 To preserve good startup time, babashka does not scan the classpath for
 `data_readers.clj` files.
 
-## Uberscript
-
-The `--uberscript` option collects the expressions in
-`BABASHKA_PRELOADS`, the command line expression or file, the main entrypoint
-and all required namespaces from the classpath into a single file. This can be
-convenient for debugging and deployment.
-
-Given the `deps.edn` from above:
-
-``` clojure
-$ deps.clj -A:my-script -Scommand "bb -cp {{classpath}} {{main-opts}} --uberscript my-script.clj"
-
-$ cat my-script.clj
-(ns my-gist-script)
-(defn -main [& args]
-  (println "Hello from gist script!"))
-(ns user (:require [my-gist-script]))
-(apply my-gist-script/-main *command-line-args*)
-
-$ bb my-script.clj
-Hello from gist script!
-```
-
-Caveat: building uberscripts works by running top-level `ns` and `require`
-forms. The rest of the code is not evaluated. Code that relies on dynamic
-requires may not work in an uberscript.
-
 ## Parsing command line arguments
 
 Babashka ships with `clojure.tools.cli`:

deps.edn

@@ -3,6 +3,7 @@
          "feature-java-time" "feature-java-nio"
          "sci/src" "babashka.curl/src" "babashka.pods/src"
          "babashka.nrepl/src"
+         "depstar/src"
          "resources" "sci/resources"],
  :deps {org.clojure/clojure {:mvn/version "1.10.2-alpha1"},
         org.clojure/tools.reader {:mvn/version "1.3.2"},
@@ -56,4 +57,4 @@
                          org.clojure/data.generators {:mvn/version "1.0.0"}
                          honeysql/honeysql {:mvn/version "1.0.444"}
                          minimallist/minimallist {:mvn/version "0.0.6"}
-                         circleci/bond {:mvn/version "0.4.0"}}}}}
+                         circleci/bond {:mvn/version "0.4.0"}}}}}

depstar

@@ -0,0 +1 @@
+Subproject commit e74b8ac05c64efb815153fbfdd2d31e3cad098cb

project.clj

@@ -8,7 +8,7 @@
   :license {:name "Eclipse Public License 1.0"
             :url "http://opensource.org/licenses/eclipse-1.0.php"}
   :source-paths ["src" "sci/src" "babashka.curl/src" "babashka.pods/src"
-                 "babashka.nrepl/src"]
+                 "babashka.nrepl/src" "depstar/src"]
   ;; for debugging Reflector.java code:
   ;; :java-source-paths ["sci/reflector/src-java"]
   :java-source-paths ["src-java"]

src/babashka/impl/classpath.clj

@@ -70,10 +70,11 @@
     (getResource loader resource-paths opts)))
 
 (defn main-ns [manifest-resource]
-  (some-> (Manifest. (io/input-stream manifest-resource))
-          (.getMainAttributes)
-          (.getValue "Main-Class")
-          (demunge)))
+  (with-open [is (io/input-stream manifest-resource)]
+    (some-> (Manifest. is)
+            (.getMainAttributes)
+            (.getValue "Main-Class")
+            (demunge))))
 
 ;;;; Scratch
 

src/babashka/main.clj

@@ -30,6 +30,7 @@
    [clojure.java.io :as io]
    [clojure.stacktrace :refer [print-stack-trace]]
    [clojure.string :as str]
+   [hf.depstar.uberjar :as uberjar]
    [sci.addons :as addons]
    [sci.core :as sci]
    [sci.impl.namespaces :as sci-namespaces]
@@ -143,6 +144,11 @@
                        (recur (next options)
                               (assoc opts-map
                                      :uberscript (first options))))
+                     ("--uberjar")
+                     (let [options (next options)]
+                       (recur (next options)
+                              (assoc opts-map
+                                     :uberjar (first options))))
                      ("-f" "--file")
                      (let [options (next options)]
                        (recur (next options)
@@ -186,7 +192,7 @@
                      (let [options (next options)]
                        (recur (next options)
                               (assoc opts-map :main (first options))))
-                     (if (some opts-map [:file :socket-repl :expressions :main])
+                     (if (some opts-map [:file :jar :socket-repl :expressions :main])
                        (assoc opts-map
                               :command-line-args options)
                        (let [trimmed-opt (str/triml opt)
@@ -197,7 +203,8 @@
                                (update :expressions (fnil conj []) (first options))
                                (assoc :command-line-args (next options)))
                            (assoc opts-map
-                                  :file opt
+                                  (if (str/ends-with? opt ".jar")
+                                    :jar :file) opt
                                   :command-line-args (next options)))))))
                  opts-map))]
     opts))
@@ -251,7 +258,7 @@ Evaluation:
   -f, --file <path>   Evaluate a file.
   -cp, --classpath    Classpath to use.
   -m, --main <ns>     Call the -main function from namespace with args.
-  --verbose           Print entire stacktrace in case of exception.
+  --verbose           Print debug information and entire stacktrace in case of exception.
 
 REPL:
 
@@ -462,7 +469,7 @@ If neither -e, -f, or --socket-repl are specified, then the first argument that
                     :repl :socket-repl :nrepl
                     :verbose? :classpath
                     :main :uberscript :describe?
-                    :jar] :as _opts}
+                    :jar :uberjar] :as _opts}
             (parse-opts args)
             _ (do ;; set properties
                 (when main (System/setProperty "babashka.main" main))
@@ -573,6 +580,7 @@ If neither -e, -f, or --socket-repl are specified, then the first argument that
                        repl [(repl/start-repl! sci-ctx) 0]
                        socket-repl [(start-socket-repl! socket-repl sci-ctx) 0]
                        nrepl [(start-nrepl! nrepl sci-ctx) 0]
+                       uberjar [nil 0]
                        expressions
                        (try
                          (loop []
@@ -601,13 +609,18 @@ If neither -e, -f, or --socket-repl are specified, then the first argument that
                 1)]
         (flush)
         (when uberscript
-          uberscript
           (let [uberscript-out uberscript]
             (spit uberscript-out "") ;; reset file
             (doseq [s (distinct @uberscript-sources)]
               (spit uberscript-out s :append true))
             (spit uberscript-out preloads :append true)
             (spit uberscript-out expression :append true)))
+        (when uberjar
+          (uberjar/run {:dest uberjar
+                        :jar :uber
+                        :classpath classpath
+                        :main-class main
+                        :verbose verbose?}))
         exit-code))))
 
 (defn -main

test-resources/babashka/uberjar/deps.edn

@@ -0,0 +1,3 @@
+{:aliases {:babashka {:classpath-overrides {org.clojure/clojure ""
+                                            org.clojure/spec.alpha ""
+                                            org.clojure/core.specs.alpha ""}}}}

test-resources/babashka/uberjar/src/my/impl.clj

@@ -0,0 +1,6 @@
+(ns my.impl
+  (:require [clojure.string]))
+
+(defn impl-fn
+  "identity"
+  [x] x)

test-resources/babashka/uberjar/src/my/impl2.clj

@@ -0,0 +1,4 @@
+(ns my.impl2
+  (:require [my.impl :as impl]))
+
+(def impl-fn impl/impl-fn)

test-resources/babashka/uberjar/src/my/main_main.clj

@@ -0,0 +1,6 @@
+(ns my.main-main
+  (:require [my.impl :as impl])
+  (:require [my.impl2 :as impl2]))
+
+(defn -main [& args]
+  (impl/impl-fn args))

test/babashka/uberjar_test.clj

@@ -0,0 +1,32 @@
+(ns babashka.uberjar-test
+  (:require
+   [babashka.test-utils :as tu]
+   [clojure.edn :as edn]
+   [clojure.string :as str]
+   [clojure.test :as t :refer [deftest is testing]]))
+
+(defn bb [input & args]
+  (edn/read-string (apply tu/bb (when (some? input) (str input)) (map str args))))
+
+(deftest uberjar-test
+  (let [tmp-file (java.io.File/createTempFile "uber" ".jar")
+        path (.getPath tmp-file)]
+    (.deleteOnExit tmp-file)
+    (testing "uberjar"
+      (tu/bb nil "--classpath" "test-resources/babashka/uberjar/src" "-m" "my.main-main" "--uberjar" path)
+      (is (= "(\"1\" \"2\" \"3\" \"4\")\n"
+             (tu/bb nil "--jar" path "1" "2" "3" "4")))
+      (is (= "(\"1\" \"2\" \"3\" \"4\")\n"
+             (tu/bb nil "-jar" path "1" "2" "3" "4")))
+      (is (= "(\"1\" \"2\" \"3\" \"4\")\n"
+             (tu/bb nil path "1" "2" "3" "4")))))
+  (testing "without main, a REPL starts"
+    ;; NOTE: if we choose the same tmp-file as above and doing this all in the
+    ;; same JVM process, the below test fails because my.main-main will be the
+    ;; main class in the manifest, even if we delete the tmp-file, which may
+    ;; indicate a state-related bug somewhere!
+    (let [tmp-file (java.io.File/createTempFile "uber" ".jar")
+          path (.getPath tmp-file)]
+      (.deleteOnExit tmp-file)
+      (tu/bb nil  "--classpath" "test-resources/babashka/uberjar/src" "--uberjar" path)
+      (is (str/includes? (tu/bb "(+ 1 2 3)" path) "6")))))

test/babashka/uberscript_test.clj

@@ -15,7 +15,6 @@
            (tu/bb nil "--file" (.getPath tmp-file) "1" "2" "3" "4")))
     (testing "order of namespaces is correct"
       (tu/bb nil "--classpath" "test-resources/babashka/uberscript/src" "-m" "my.main" "--uberscript" (.getPath tmp-file))
-      (spit "/tmp/foo.clj" (slurp (.getPath tmp-file)))
       (is (= "(\"1\" \"2\" \"3\" \"4\")\n"
              (tu/bb nil "--file" (.getPath tmp-file) "1" "2" "3" "4"))))))