luciano/babashka
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1190: task dependencies resolve as core.async channels (#1191)

Browse source
This commit is contained in:
Michiel Borkent 2022-02-27 11:49:40 +01:00 committed by GitHub
parent 5c6e2b5167
commit 141881376d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 109 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
deps.edn
View file

@ -124,7 +124,6 @@
io.lambdaforge/datalog-parser {:mvn/version "0.1.9"} io.lambdaforge/datalog-parser {:mvn/version "0.1.9"}
clj-stacktrace/clj-stacktrace {:mvn/version "0.2.8"} clj-stacktrace/clj-stacktrace {:mvn/version "0.2.8"}
clojure-msgpack/clojure-msgpack {:mvn/version "1.2.1"} clojure-msgpack/clojure-msgpack {:mvn/version "1.2.1"}
cli-matic {:git/url "https://github.com/l3nz/cli-matic.git", :git/sha "9cd53ba7336363e3d06650dbad413b6f8b06e471"}
cli-matic/cli-matic {:git/url "https://github.com/l3nz/cli-matic.git", :git/sha "9cd53ba7336363e3d06650dbad413b6f8b06e471"}} cli-matic/cli-matic {:git/url "https://github.com/l3nz/cli-matic.git", :git/sha "9cd53ba7336363e3d06650dbad413b6f8b06e471"}}
:classpath-overrides {org.clojure/clojure nil :classpath-overrides {org.clojure/clojure nil
org.clojure/spec.alpha nil}} org.clojure/spec.alpha nil}}

57
src/babashka/impl/tasks.clj
View file

@ -152,25 +152,40 @@
(format "(babashka.tasks/-wait %s)" task-name) (format "(babashka.tasks/-wait %s)" task-name)
task-name)))) task-name))))
(def o (Object.))
#_:clj-kondo/ignore
(defn- log
"Used internally for debugging"
[& strs]
(locking o
(apply prn strs)))
(defn wait-tasks [deps] (defn wait-tasks [deps]
(if deps (if deps
(format " (format
(let [chans (filter babashka.tasks/-chan? %s)] (pr-str
(loop [cs chans] '(let [chans (filter babashka.tasks/-chan? %s)]
(when (seq cs) (loop [cs chans]
(let [[v p] (clojure.core.async/alts!! cs) (when (seq cs)
[task-name v] v (let [[v* p] (clojure.core.async/alts!! cs)
cs (filterv #(not= p %%) cs) [task-name v] v*
;; _ (.println System/err (str \"n: \" task-name \" v: \" v)) cs (filterv #(not= p %) cs)
;; check for existence of v, as the channel may already have been consumed once _ (when v* (intern *ns* (symbol task-name) v))]
_ (when v (intern *ns* (symbol task-name) v))] (when (instance? Throwable v)
(when (instance? Throwable v) (throw (ex-info (ex-message v)
(throw (ex-info (ex-message v) {:babashka/exit 1
{:babashka/exit 1 :data (ex-data v)})))
:data (ex-data v)}))) (recur cs))))
(recur cs)))))" deps) ;; since resolving channels into values may happen in parallel and some
"") ;; channels may have been resolved on other threads, we should wait
#_(format "(def %s (babashka.tasks/-wait %s))" dep dep)) ;; until all deps have been interned as values rather than chans
;; see issue 1190
(loop [deps '%s]
(when (some (fn [task-name]
(babashka.tasks/-chan? (deref (resolve (symbol task-name))))) deps)
(recur deps))))) deps deps)
""))
(defn wrap-enter-leave [task-name prog enter leave] (defn wrap-enter-leave [task-name prog enter leave]
(str (pr-str enter) "\n" (str (pr-str enter) "\n"
@ -184,7 +199,8 @@
(defn wrap-depends [prog depends parallel?] (defn wrap-depends [prog depends parallel?]
(if parallel? (if parallel?
(format "(do %s)" (str (str "\n" (wait-tasks depends)) "\n" prog)) (format "(do %s)" (str (str "\n" (wait-tasks depends))
"\n" prog))
prog)) prog))
(defn assemble-task-1 (defn assemble-task-1
@ -231,7 +247,7 @@
%s ;; deps %s ;; deps
(ns %s %s) (ns %s %s)
(require '[babashka.tasks]) (require '[babashka.tasks #_#_:refer [log]])
(when-not (resolve 'clojure) (when-not (resolve 'clojure)
;; we don't use refer so users can override this ;; we don't use refer so users can override this
(intern *ns* 'clojure babashka.tasks/clojure)) (intern *ns* 'clojure babashka.tasks/clojure))
@ -441,4 +457,5 @@
'*task* task '*task* task
'current-task current-task 'current-task current-task
'current-state state 'current-state state
'run (sci/copy-var run sci-ns)}) 'run (sci/copy-var run sci-ns)
#_#_'log log})

101
test/babashka/bb_edn_test.clj
View file

@ -80,7 +80,7 @@
fix-lines test-utils/normalize] fix-lines test-utils/normalize]
(testing "shell test" (testing "shell test"
(test-utils/with-config {:tasks {'foo (list 'shell {:out out} (test-utils/with-config {:tasks {'foo (list 'shell {:out out}
echo-cmd "hello")}} echo-cmd "hello")}}
(bb "foo") (bb "foo")
(is (= "hello\n" (fix-lines (slurp out)))))) (is (= "hello\n" (fix-lines (slurp out))))))
(fs/delete out) (fs/delete out)
@ -89,30 +89,30 @@
:err out :err out
:continue '(fn [proc] :continue '(fn [proc]
(contains? proc :exit))} (contains? proc :exit))}
ls-cmd "foobar") ls-cmd "foobar")
:exit)}} :exit)}}
(is (pos? (bb "run" "--prn" "foo"))))) (is (pos? (bb "run" "--prn" "foo")))))
(testing "shell test with :error" (testing "shell test with :error"
(test-utils/with-config (test-utils/with-config
{:tasks {'foo (list '-> (list 'shell {:out out {:tasks {'foo (list '-> (list 'shell {:out out
:err out :err out
:error-fn '(constantly 1337)} :error-fn '(constantly 1337)}
ls-cmd "foobar"))}} ls-cmd "foobar"))}}
(is (= 1337 (bb "run" "--prn" "foo")))) (is (= 1337 (bb "run" "--prn" "foo"))))
(test-utils/with-config (test-utils/with-config
{:tasks {'foo (list '-> (list 'shell {:out out {:tasks {'foo (list '-> (list 'shell {:out out
:err out :err out
:error-fn :error-fn
'(fn [opts] '(fn [opts]
(and (:task opts) (and (:task opts)
(:proc opts) (:proc opts)
(not (zero? (:exit (:proc opts))))))} (not (zero? (:exit (:proc opts))))))}
ls-cmd "foobar"))}} ls-cmd "foobar"))}}
(is (true? (bb "run" "--prn" "foo"))))) (is (true? (bb "run" "--prn" "foo")))))
(fs/delete out) (fs/delete out)
(testing "clojure test" (testing "clojure test"
(test-utils/with-config {:tasks {'foo (list 'clojure {:out out} (test-utils/with-config {:tasks {'foo (list 'clojure {:out out}
"-M -e" "(println :yolo)")}} "-M -e" "(println :yolo)")}}
(bb "foo") (bb "foo")
(is (= ":yolo\n" (fix-lines (slurp out)))))) (is (= ":yolo\n" (fix-lines (slurp out))))))
(fs/delete out) (fs/delete out)
@ -179,27 +179,27 @@
(testing "no such task" (testing "no such task"
(test-utils/with-config '{:tasks {a (+ 1 2 3)}} (test-utils/with-config '{:tasks {a (+ 1 2 3)}}
(is (thrown-with-msg? (is (thrown-with-msg?
Exception #"No such task: b" Exception #"No such task: b"
(bb "run" "b"))))) (bb "run" "b")))))
(testing "unresolved dependency" (testing "unresolved dependency"
(test-utils/with-config '{:tasks {a (+ 1 2 3) (test-utils/with-config '{:tasks {a (+ 1 2 3)
b {:depends [x] b {:depends [x]
:task (+ a 4 5 6)}}} :task (+ a 4 5 6)}}}
(is (thrown-with-msg? (is (thrown-with-msg?
Exception #"No such task: x" Exception #"No such task: x"
(bb "run" "b"))))) (bb "run" "b")))))
(testing "cyclic task" (testing "cyclic task"
(test-utils/with-config '{:tasks {b {:depends [b] (test-utils/with-config '{:tasks {b {:depends [b]
:task (+ a 4 5 6)}}} :task (+ a 4 5 6)}}}
(is (thrown-with-msg? (is (thrown-with-msg?
Exception #"Cyclic task: b" Exception #"Cyclic task: b"
(bb "run" "b")))) (bb "run" "b"))))
(test-utils/with-config '{:tasks {c {:depends [b]} (test-utils/with-config '{:tasks {c {:depends [b]}
b {:depends [c] b {:depends [c]
:task (+ a 4 5 6)}}} :task (+ a 4 5 6)}}}
(is (thrown-with-msg? (is (thrown-with-msg?
Exception #"Cyclic task: b" Exception #"Cyclic task: b"
(bb "run" "b"))))) (bb "run" "b")))))
(testing "doc" (testing "doc"
(test-utils/with-config '{:tasks {b {:doc "Beautiful docstring"}}} (test-utils/with-config '{:tasks {b {:doc "Beautiful docstring"}}}
(let [s (test-utils/bb nil "doc" "b")] (let [s (test-utils/bb nil "doc" "b")]
@ -232,7 +232,7 @@
(throw (ex-info "0 noes" {}))) (throw (ex-info "0 noes" {})))
c {:depends [a b]}}} c {:depends [a b]}}}
(is (thrown-with-msg? Exception #"0 noes" (is (thrown-with-msg? Exception #"0 noes"
(bb "run" "--parallel" "c"))))) (bb "run" "--parallel" "c")))))
(testing "edge case" (testing "edge case"
(test-utils/with-config '{:tasks (test-utils/with-config '{:tasks
{a (run '-a {:parallel true}) {a (run '-a {:parallel true})
@ -265,7 +265,7 @@
*server* server] *server* server]
(babashka.tasks/run 'server)))}} (babashka.tasks/run 'server)))}}
(is (= '([8 :foo] [8 :bar] [11 :foo] [11 :bar] [15 :foo] [15 :bar]) (is (= '([8 :foo] [8 :bar] [11 :foo] [11 :bar] [15 :foo] [15 :bar])
(bb "run" "--prn" "run-all"))))) (bb "run" "--prn" "run-all")))))
;; TODO: disabled because of " Volume in drive C has no label.\r\n Volume Serial Number is 1CB8-D4AA\r\n\r\n Directory of C:\\projects\\babashka\r\n\r\n" on Appveyor. See https://ci.appveyor.com/project/borkdude/babashka/builds/40003094. ;; TODO: disabled because of " Volume in drive C has no label.\r\n Volume Serial Number is 1CB8-D4AA\r\n\r\n Directory of C:\\projects\\babashka\r\n\r\n" on Appveyor. See https://ci.appveyor.com/project/borkdude/babashka/builds/40003094.
(testing "shell test with :continue" (testing "shell test with :continue"
(let [ls-cmd (if main/windows? "cmd /c dir" "ls")] (let [ls-cmd (if main/windows? "cmd /c dir" "ls")]
@ -278,22 +278,22 @@
(deftest ^:skip-windows unix-task-test (deftest ^:skip-windows unix-task-test
(testing "shell pipe test" (testing "shell pipe test"
(test-utils/with-config '{:tasks {a (-> (shell {:out :string} (test-utils/with-config '{:tasks {a (-> (shell {:out :string}
"echo hello") "echo hello")
(shell {:out :string} "cat") (shell {:out :string} "cat")
:out)}} :out)}}
(let [s (bb "run" "--prn" "a")] (let [s (bb "run" "--prn" "a")]
(is (= "hello\n" s)))))) (is (= "hello\n" s))))))
(deftest ^:windows-only win-task-test (deftest ^:windows-only win-task-test
(when main/windows? (when main/windows?
(testing "shell pipe test" (testing "shell pipe test"
; this task prints the contents of deps.edn ; this task prints the contents of deps.edn
(test-utils/with-config '{:tasks {a (->> (shell {:out :string} (test-utils/with-config '{:tasks {a (->> (shell {:out :string}
"cmd /c echo deps.edn") "cmd /c echo deps.edn")
:out :out
clojure.string/trim-newline clojure.string/trim-newline
(shell {:out :string} "cmd /c type") (shell {:out :string} "cmd /c type")
:out)}} :out)}}
(let [s (bb "run" "--prn" "a")] (let [s (bb "run" "--prn" "a")]
(is (str/includes? s "paths"))))))) (is (str/includes? s "paths")))))))
@ -373,3 +373,46 @@ even more stuff here\"
(testing "bb.edn without :deps should not require deps.clj" (testing "bb.edn without :deps should not require deps.clj"
(test-utils/with-config '{:tasks {a 1}} (test-utils/with-config '{:tasks {a 1}}
(bb "-e" "(+ 1 2 3)")))))) (bb "-e" "(+ 1 2 3)"))))))
(deftest deps-race-condition-test
(test-utils/with-config
(pr-str '{:tasks {task-b (do
(Thread/sleep 10)
:task00-out)
task-c {:depends [task-b]
:task (do
(println
"task-b: "
(type task-b))
{})}
task-a {:task (do
(Thread/sleep 10)
:task0-out)}
task-e {:depends [task-e1] :task {}}
task-e2 {:depends [task-a] :task {}}
task-e3 {:depends [task-b] :task {}}
task-e1 {:depends [task-e2 task-e3]
:task {}}
task-h {:depends [task-a task-b]
:task {}}
task-d {:task (do (Thread/sleep 2) {})}
task-f {:depends [task-d task-e task-a]
:task {}}
task-g {:depends [task-f
task-d
task-a
task-c
task-h]
:task {}}}})
(time (dotimes [_ 50]
(is (str/includes? (test-utils/bb nil "run" "--parallel" "task-g")
"task-b: clojure.lang.Keyword"))))))
(deftest parallel-nil-results-test
(test-utils/with-config
(pr-str '{:tasks {a (do nil)
b (do nil)
c (do nil)
d {:depends [a b c]
:task (prn [a b c])}}})
(is (= [nil nil nil] (bb "run" "--parallel" "d")))))