[#1110] Load tasks and deps from other bb.edn file (#1117)

Co-authored-by: Bob <highpressurecarsalesman@gmail.com>

.circleci/config.yml

@@ -58,8 +58,8 @@ jobs:
 
             java -jar $jar script/reflection.clj
             reflection="babashka-$VERSION-reflection.json"
-            BABASHKA_EDN=".build/bb.edn" java -jar "$jar" release-artifact "$jar"
+            java -jar "$jar" --config .build/bb.edn --deps-root . release-artifact "$jar"
-            BABASHKA_EDN=".build/bb.edn" java -jar "$jar" release-artifact "$reflection"
+            java -jar "$jar" --config .build/bb.edn --deps-root . release-artifact "$reflection"
       - store_artifacts:
           path: /tmp/release
           destination: release

.circleci/script/release

@@ -24,7 +24,7 @@ tar zcvf "$archive" bb # bbk
 
 cd -
 
-BABASHKA_EDN=".build/bb.edn" ./bb release-artifact "/tmp/release/$archive"
+./bb --config .build/bb.edn --deps-root . release-artifact "/tmp/release/$archive"
 
 ## cleanup
 

.clj-kondo/babashka/fs/config.edn

@@ -0,0 +1 @@
+{:lint-as {babashka.fs/with-temp-dir clojure.core/let}}

.dir-locals.el

@@ -0,0 +1,2 @@
+((clojure-mode
+  (cider-clojure-cli-aliases . "test")))

appveyor.yml

@@ -62,11 +62,7 @@ build_script:
 
     jar -cMf %zip% bb.exe
 
-    set BABASHKA_EDN=.build/bb.edn
+    bb --config .build/bb.edn --deps-root . release-artifact %zip%
-
-    bb release-artifact %zip%
-
-    set BABASHKA_EDN=
 
     set BABASHKA_CLASSPATH=
 

script/lib_tests/run_all_libtests.bat

@@ -4,8 +4,4 @@ set EDN=lib_tests.edn
 
 .\bb -f script/lib_tests/bb_edn_from_deps.clj %EDN%
 
-set BABASHKA_EDN=%EDN%
+%BB_CMD% --config %EDN% --deps-root . -f test-resources/lib_tests/babashka/run_all_libtests.clj %*
-
-%BB_CMD% -f test-resources/lib_tests/babashka/run_all_libtests.clj %*
-
-set BABASHKA_EDN=

src/babashka/impl/deps.clj

@@ -1,6 +1,8 @@
 (ns babashka.impl.deps
   (:require [babashka.deps :as bdeps]
+            [babashka.fs :as fs]
             [babashka.impl.classpath :as cp]
+            [babashka.impl.common :refer [bb-edn]]
             [borkdude.deps :as deps]
             [clojure.string :as str]
             [sci.core :as sci]))
@@ -58,8 +60,19 @@
   ([deps-map] (add-deps deps-map nil))
   ([deps-map {:keys [:aliases :env :extra-env :force]}]
    (when-let [paths (:paths deps-map)]
-     (cp/add-classpath (str/join cp/path-sep paths)))
+     (let [paths (let [deps-root (:deps-root @bb-edn)
-   (when-let [deps-map (not-empty (dissoc deps-map :paths :tasks :raw :min-bb-version))]
+                       deps-root (fs/absolutize deps-root)
+                       ;; cwd (fs/absolutize ".")
+                       ;; rel (fs/relativize cwd f)
+                       paths (mapv #(str (fs/file deps-root %)) paths)]
+                   paths)]
+       (cp/add-classpath (str/join cp/path-sep paths))))
+   (when-let [deps-map (not-empty (dissoc deps-map
+                                          ;; paths are added manually above
+                                          :paths
+                                          ;; extra-paths are transformed to :deps in task handling
+                                          :extra-paths
+                                          :tasks :raw :min-bb-version))]
      (binding [*print-namespace-maps* false]
        (let [deps-map (assoc-in deps-map [:aliases :org.babashka/defaults]
                                 {:replace-paths [] ;; babashka sets paths manually

src/babashka/impl/tasks.clj

@@ -229,8 +229,7 @@
 
 (defn format-task [init extra-paths extra-deps requires prog]
   (format "
-%s ;; extra-paths
+%s ;; deps
-%s ;; extra-deps
 
 (ns %s %s)
 (require '[babashka.tasks])
@@ -251,12 +250,12 @@
 %s
 
 "
-          (if (seq extra-paths)
+          (let [deps (cond-> {}
-            (format "(babashka.classpath/add-classpath \"%s\")" (str/join cp/path-sep extra-paths))
+                       (seq extra-deps) (assoc :deps extra-deps)
-            "")
+                       (seq extra-paths) (assoc :paths extra-paths))]
-          (if (seq extra-deps)
+            (if (seq deps)
-            (format "(babashka.deps/add-deps '%s)" (pr-str {:deps extra-deps}))
+              (format "(babashka.deps/add-deps '%s)" (pr-str deps))
-            "")
+              ""))
           @rand-ns
           (if (seq requires)
             (format "(:require %s)" (str/join " " requires))
@@ -271,13 +270,13 @@
          depends (:depends task)]
      (when (contains? processing task-name)
        (throw (ex-info (str "Cyclic task: " task-name) {})))
-     (loop [deps (seq depends)]
+     (let [deps (seq depends)
-       (let [deps (remove #(contains? @processed %) deps)
+           deps (remove #(contains? @processed %) deps)
            order (vec (mapcat #(target-order tasks % processed (conj processing task-name)) deps))]
        (if-not (contains? @processed task-name)
          (do (vswap! processed conj task-name)
              (conj order task-name))
-           order))))))
+         order)))))
 
 #_(defn tasks->dependees [task-names tasks]
     (let [tasks->depends (zipmap task-names (map #(:depends (get tasks %)) task-names))]

src/babashka/main.clj

@@ -137,6 +137,8 @@ Global opts:
   --debug           Print debug information and internal stacktrace in case of exception.
   --force           Passes -Sforce to deps.clj, forcing recalculation of the classpath.
   --init <file>     Load file after any preloads and prior to evaluation/subcommands.
+  --config <file>   Replacing bb.edn with file. Relative paths are resolved relative to file.
+  --deps-root <dir> Treat dir as root of relative paths in config.
 
 Help:
 
@@ -632,21 +634,27 @@ Use bb run --help to show this help output.
     (if options
       (case (first options)
         ("--classpath" "-cp") (recur (nnext options) (assoc opts-map :classpath (second options)))
+
         ("--debug"
-         "--verbose" ;; renamed to --debug
+         "--verbose")
-         ) (recur (next options) (assoc opts-map :debug true))
+        ;; renamed to --debug
+        (recur (next options) (assoc opts-map :debug true))
+
         ("--init")
         (recur (nnext options) (assoc opts-map :init (second options)))
+
+        ("--config")
+        (recur (nnext options) (assoc opts-map :config (second options)))
+
+        ("--deps-root")
+        (recur (nnext options) (assoc opts-map :deps-root (second options)))
         [options opts-map])
       [options opts-map])))
 
 (defn parse-opts
   ([options] (parse-opts options nil))
   ([options opts-map]
-   (let [[options opts-map] (if opts-map
+   (let [opt (first options)
-                              [options opts-map]
-                              (parse-global-opts options))
-         opt (first options)
          tasks (into #{} (map str) (keys (:tasks @common/bb-edn)))]
      (if-not opt opts-map
              ;; FILE > TASK > SUBCOMMAND
@@ -902,59 +910,62 @@ Use bb run --help to show this help output.
         exit-code))))
 
 (defn satisfies-min-version? [min-version]
-(let [[major-current minor-current patch-current] version-data
+  (let [[major-current minor-current patch-current] version-data
         [major-min minor-min patch-min] (parse-version min-version)]
-(or (> major-current major-min)
+    (or (> major-current major-min)
         (and (= major-current major-min)
              (or (> minor-current minor-min)
                  (and (= minor-current minor-min)
                       (>= patch-current patch-min)))))))
 
 (defn main [& args]
-(let [bb-edn-file (or (System/getenv "BABASHKA_EDN")
+  (let [[args global-opts] (parse-global-opts args)
+        bb-edn-file (or (:config global-opts)
                         "bb.edn")
-      bb-edn (or (when (fs/exists? bb-edn-file)
+        bb-edn (when (fs/exists? bb-edn-file)
                  (let [raw-string (slurp bb-edn-file)
                        edn (edn/read-string raw-string)
-                         edn (assoc edn :raw raw-string)]
+                       edn (assoc edn
+                                  :raw raw-string
+                                  :file bb-edn-file
+                                  :deps-root
+                                  (or (:deps-root global-opts)
+                                      (str (fs/parent bb-edn-file))))]
                    (vreset! common/bb-edn edn)))
-                 ;; tests may have modified bb-edn
-                 @common/bb-edn)
         min-bb-version (:min-bb-version bb-edn)]
-(when min-bb-version
+    (when min-bb-version
       (when-not (satisfies-min-version? min-bb-version)
         (binding [*out* *err*]
           (println (str "WARNING: this project requires babashka "
-                    min-bb-version " or newer, but you have: " version))))))
+                        min-bb-version " or newer, but you have: " version)))))
-(let [opts (parse-opts args)]
+    (exec (parse-opts args global-opts))))
-(exec opts)))
 
 (def musl?
-"Captured at compile time, to know if we are running inside a
+  "Captured at compile time, to know if we are running inside a
   statically compiled executable with musl."
-(and (= "true" (System/getenv "BABASHKA_STATIC"))
+  (and (= "true" (System/getenv "BABASHKA_STATIC"))
-(= "true" (System/getenv "BABASHKA_MUSL"))))
+       (= "true" (System/getenv "BABASHKA_MUSL"))))
 
 (defmacro run [args]
-(if musl?
+  (if musl?
-;; When running in musl-compiled static executable we lift execution of bb
+    ;; When running in musl-compiled static executable we lift execution of bb
-;; inside a thread, so we have a larger than default stack size, set by an
+    ;; inside a thread, so we have a larger than default stack size, set by an
-;; argument to the linker. See https://github.com/oracle/graal/issues/3398
+    ;; argument to the linker. See https://github.com/oracle/graal/issues/3398
-`(let [v# (volatile! nil)
+    `(let [v# (volatile! nil)
            f# (fn []
                 (vreset! v# (apply main ~args)))]
        (doto (Thread. nil f# "main")
          (.start)
          (.join))
        @v#)
-`(apply main ~args)))
+    `(apply main ~args)))
 
 (defn -main
-[& args]
+  [& args]
-(handle-pipe!)
+  (handle-pipe!)
-(handle-sigint!)
+  (handle-sigint!)
-(if-let [dev-opts (System/getenv "BABASHKA_DEV")]
+  (if-let [dev-opts (System/getenv "BABASHKA_DEV")]
-(let [{:keys [:n]} (if (= "true" dev-opts) {:n 1}
+    (let [{:keys [:n]} (if (= "true" dev-opts) {:n 1}
                            (edn/read-string dev-opts))
           last-iteration (dec n)]
       (dotimes [i n]
@@ -963,7 +974,7 @@ Use bb run --help to show this help output.
           (do (run args)
               (binding [*out* *err*]
                 (println "ran" n "times"))))))
-(let [exit-code (run args)]
+    (let [exit-code (run args)]
       (System/exit exit-code))))
 
 ;;;; Scratch

test-resources/lib_tests/clojure/data/json_test.clj

@@ -414,7 +414,7 @@
     (is (= x (json/read-str (with-out-str (json/pprint x)))))))
 
 (deftest pretty-print-nonescaped-unicode
-  (is (= "\"\u1234\u4567\"\n"
+  (is (= (str "\"\u1234\u4567\"" (System/lineSeparator))
          (with-out-str
            (json/pprint "\u1234\u4567" :escape-unicode false)))))
 

test-resources/lib_tests/table/core_test.clj

@@ -15,7 +15,7 @@
       +---+---+
       | 3 | 4 |
       +---+---+
-      ") "\n")
+      ") (System/lineSeparator))
     (with-out-str (table [["1" "2"] ["3" "4"]])))))
 
 (deftest test-table-with-vecs-in-vec

test/babashka/bb_edn_test.clj

@@ -1,12 +1,14 @@
 (ns babashka.bb-edn-test
   (:require
    [babashka.fs :as fs]
+   [babashka.impl.classpath :as cp]
    [babashka.impl.common :as common]
    [babashka.main :as main]
    [babashka.test-utils :as test-utils]
    [clojure.edn :as edn]
    [clojure.string :as str]
-   [clojure.test :as test :refer [deftest is testing]]))
+   [clojure.test :as test :refer [deftest is testing]]
+   [sci.core :as sci]))
 
 (defn bb [& args]
   (let [args (map str args)
@@ -220,7 +222,10 @@
                 t1             (System/currentTimeMillis)
                 delta-parallel (- t1 t0)]
             (is (= tree s))
-            (is (< delta-parallel delta-sequential))))))
+            (when (>=  (doto (-> (Runtime/getRuntime) (.availableProcessors))
+                         (prn))
+                       2)
+              (is (< delta-parallel delta-sequential)))))))
     (testing "exception"
       (test-utils/with-config '{:tasks {a (Thread/sleep 10000)
                                         b (do (Thread/sleep 10)
@@ -337,20 +342,24 @@
         (is (= "uberjar" (:file (main/parse-opts ["uberjar"]))))
         (finally (fs/delete "uberjar"))))))
 
-(deftest min-bb-version
+(deftest min-bb-version-test
-  (when-not test-utils/native?
+  (fs/with-temp-dir [dir {}]
-    (vreset! common/bb-edn '{:min-bb-version "300.0.0"})
+    (let [config (str (fs/file dir "bb.edn"))]
+      (spit config '{:min-bb-version "300.0.0"})
       (let [sw (java.io.StringWriter.)]
         (binding [*err* sw]
-        (main/main "-e" "nil"))
+          (main/main "--config" config  "-e" "nil"))
         (is (str/includes? (str sw)
-                         "WARNING: this project requires babashka 300.0.0 or newer, but you have: ")))))
+                           "WARNING: this project requires babashka 300.0.0 or newer, but you have: "))))))
 
-;; TODO:
+(deftest classpath-other-bb-edn-test
-;; Do we want to support the same parsing as the clj CLI?
+  (fs/with-temp-dir [dir {}]
-;; Or do we want `--aliases :foo:bar`
+    (let [config (str (fs/file dir "bb.edn"))]
-;; Let's wait for a good use case
+      (spit config '{:paths ["src"]
-#_(deftest alias-deps-test
+                     :tasks {cp (prn (babashka.classpath/get-classpath))}})
-    (test-utils/with-config '{:aliases {:medley {:deps {medley/medley {:mvn/version "1.3.0"}}}}}
+      (let [out (bb "--config" config "cp")
-      (is (= '{1 {:id 1}, 2 {:id 2}}
+            entries (cp/split-classpath out)
-             (bb "-A:medley" "-e" "(require 'medley.core)" "-e" "(medley.core/index-by :id [{:id 1} {:id 2}])")))))
+            entry (first entries)]
+        (is (= 1 (count entries)))
+        (is (= (fs/parent config) (fs/parent entry)))
+        (is (str/ends-with? entry "src"))))))

test/babashka/main_test.clj

@@ -18,30 +18,34 @@
      :eof nil}
     (apply test-utils/bb (when (some? input) (str input)) (map str args)))))
 
+(defn parse-opts [args]
+  (let [[args global-opts] (main/parse-global-opts args)]
+    (main/parse-opts args global-opts)))
+
 (deftest parse-opts-test
   (is (= "1667"
-         (:nrepl (main/parse-opts ["--nrepl-server"]))))
+         (:nrepl (parse-opts ["--nrepl-server"]))))
   (is (= "1666"
-         (:socket-repl (main/parse-opts ["--socket-repl"]))))
+         (:socket-repl (parse-opts ["--socket-repl"]))))
   (is (= {:nrepl "1667", :classpath "src"}
-         (main/parse-opts ["--nrepl-server" "-cp" "src"])))
+         (parse-opts ["--nrepl-server" "-cp" "src"])))
   (is (= {:nrepl "1667", :classpath "src"}
-         (main/parse-opts ["-cp" "src" "nrepl-server"])))
+         (parse-opts ["-cp" "src" "nrepl-server"])))
   (is (= {:socket-repl "1666", :expressions ["123"]}
-         (main/parse-opts ["--socket-repl" "-e" "123"])))
+         (parse-opts ["--socket-repl" "-e" "123"])))
   (is (= {:socket-repl "1666", :expressions ["123"]}
-         (main/parse-opts ["--socket-repl" "1666" "-e" "123"])))
+         (parse-opts ["--socket-repl" "1666" "-e" "123"])))
   (is (= {:nrepl "1666", :expressions ["123"]}
-         (main/parse-opts ["--nrepl-server" "1666" "-e" "123"])))
+         (parse-opts ["--nrepl-server" "1666" "-e" "123"])))
   (is (= {:classpath "src"
           :uberjar "foo.jar"}
-         (main/parse-opts ["--classpath" "src" "uberjar" "foo.jar"])))
+         (parse-opts ["--classpath" "src" "uberjar" "foo.jar"])))
   (is (= {:classpath "src"
           :uberjar "foo.jar"
           :debug true}
-         (main/parse-opts ["--debug" "--classpath" "src" "uberjar" "foo.jar"])))
+         (parse-opts ["--debug" "--classpath" "src" "uberjar" "foo.jar"])))
-  (is (= "src" (:classpath (main/parse-opts ["--classpath" "src"]))))
+  (is (= "src" (:classpath (parse-opts ["--classpath" "src"]))))
-  (is (:debug (main/parse-opts ["--debug"])))
+  (is (:debug (parse-opts ["--debug"])))
   (is (= 123 (bb nil "(println 123)")))
   (is (= 123 (bb nil "-e" "(println 123)")))
   (is (= 123 (bb nil "--eval" "(println 123)")))
@@ -54,8 +58,8 @@
   (let [v (bb nil "--describe")]
     (is (:babashka/version v))
     (is (:feature/xml v)))
-  (is (= {:force? true} (main/parse-opts ["--force"])))
+  (is (= {:force? true} (parse-opts ["--force"])))
-  (is (= {:main "foo", :command-line-args '("-h")} (main/parse-opts ["-m" "foo" "-h"]))))
+  (is (= {:main "foo", :command-line-args '("-h")} (parse-opts ["-m" "foo" "-h"]))))
 
 (deftest version-test
   (is (= [1 0 0] (main/parse-version "1.0.0-SNAPSHOT")))

test/babashka/test_utils.clj

@@ -47,13 +47,10 @@
 (defn bb-jvm [input-or-opts & args]
   (reset! cp/cp-state nil)
   (reset! main/env {})
-  (if-let [path *bb-edn-path*]
+  (vreset! common/bb-edn nil)
-    (let [raw (slurp path)]
+  (let [args (cond-> args *bb-edn-path*
-      (vreset! common/bb-edn
+                     (->> (list* "--config" *bb-edn-path* "--deps-root" ".")))
-               (assoc (edn/read-string raw)
+        os (java.io.StringWriter.)
-                      :raw raw)))
-    (vreset! common/bb-edn nil))
-  (let [os (java.io.StringWriter.)
         es (if-let [err (:err input-or-opts)]
              err (java.io.StringWriter.))
         in (if (string? input-or-opts)
@@ -76,7 +73,9 @@
                         (apply main/main args)))]
             (if (zero? res)
               (do
-                (println (str es)) ;; flush stderr
+                (let [err (str es)]
+                  (when-not (str/blank? err)
+                    (println err))) ;; flush stderr
                 (normalize (str os)))
               (do
                 (println (str os))
@@ -89,14 +88,12 @@
         (vars/bindRoot sci/err *err*)))))
 
 (defn bb-native [input & args]
-  (let [res (p/process (into ["./bb"] args)
+  (let [args (cond-> args *bb-edn-path*
-                       (cond-> {:in input
+               (->> (list* "--config" *bb-edn-path* "--deps-root" ".")))
+        res (p/process (into ["./bb"] args)
+                       {:in input
                         :out :string
-                                :err :string}
+                        :err :string})
-                         *bb-edn-path*
-                         (assoc
-                          :extra-env (assoc (into {} (System/getenv))
-                                            "BABASHKA_EDN" *bb-edn-path*))))
         res (deref res)
         exit (:exit res)
         error? (pos? exit)]

test/babashka/uberjar_test.clj

@@ -45,16 +45,6 @@
         (is (= "(\"42\")\n" (tu/bb nil "--jar" path "-m" "my.main-main" "42")))
         (is (= "(\"42\")\n" (tu/bb nil "--classpath" path "-m" "my.main-main" "42")))
         (is (= "(\"42\")\n" (tu/bb nil path "42"))))))
-
-  ; this test fails the windows native test in CI
-  (when-not main/windows?
-    (testing "throw on empty classpath"
-      (let [tmp-file (java.io.File/createTempFile "uber" ".jar")
-            path     (.getPath tmp-file)]
-        (.deleteOnExit tmp-file)
-        (is (thrown-with-msg?
-              Exception #"classpath"
-              (tu/bb nil "uberjar" path "-m" "my.main-main"))))))
   (testing "ignore empty entries on classpath"
     (let [tmp-file (java.io.File/createTempFile "uber" ".jar")
           path (.getPath tmp-file)
@@ -63,3 +53,14 @@
       (tu/bb nil "--classpath" empty-classpath "uberjar" path "-m" "my.main-main")
       ;; Only a manifest entry is added
       (is (< (count-entries path) 3)))))
+
+(deftest throw-on-empty-classpath
+  ;; this test fails the windows native test in CI
+  (when-not main/windows?
+    (testing "throw on empty classpath"
+      (let [tmp-file (java.io.File/createTempFile "uber" ".jar")
+            path     (.getPath tmp-file)]
+        (.deleteOnExit tmp-file)
+        (is (thrown-with-msg?
+             Exception #"classpath"
+             (tu/bb nil "uberjar" path "-m" "my.main-main")))))))