From d1ea9f8360b1b2552625a5ec563ec48177bbdb37 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Fri, 16 Aug 2019 22:22:58 +0200 Subject: [PATCH] [#14] implement --stream option --- README.md | 8 +++-- script/compile | 2 ++ src/babashka/main.clj | 72 ++++++++++++++++++++++--------------- test/babashka/main_test.clj | 14 +++++++- 4 files changed, 63 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index e72956e1..decbdafb 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ You may also download a binary from [Github](https://github.com/borkdude/babashk ## Usage ``` shellsession -bb [ --help ] | [ --version ] | ( [ -i ] [ -o ] | [ -io ] ) ( expression | -f ) +bb [ --help ] | [ --version ] | ( [ -i ] [ -o ] | [ -io ] ) [ --stream ] ( expression | -f ) ``` Type `bb --help` to see a full explanation of the options. @@ -84,7 +84,8 @@ The input is read as EDN by default. If the `-i` flag is provided, then the input is read as a string which is then split on newlines. The output is printed as EDN by default, unless the `-o` flag is provided, then the output is turned into shell-scripting friendly output. To combine `-i` and `-o` you can use -`-io`. +`-io`. When using the `--stream` option the expression is executed for every +line or EDN value from stdin. The `clojure.core` functions are accessible without a namespace alias. @@ -102,7 +103,8 @@ From Java the following is available: Special vars: -- `*in*`: contains the input read from stdin +- `*in*`: contains the input read from stdin (EDN by default, multiple lines with the `-i` option) + - `*command-line-args*`: contain the command line args Examples: diff --git a/script/compile b/script/compile index 489a88a8..c4cc6ea2 100755 --- a/script/compile +++ b/script/compile @@ -11,6 +11,8 @@ fi BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION) +# We also need to AOT sci, else something didn't work in the Mac build on CircleCI +# See https://github.com/oracle/graal/issues/1613 ( cd /tmp; git clone https://github.com/borkdude/sci 2> /dev/null || true ) mkdir -p src/sci cp -R /tmp/sci/src/* src diff --git a/src/babashka/main.clj b/src/babashka/main.clj index e3f742cb..873450b4 100644 --- a/src/babashka/main.clj +++ b/src/babashka/main.clj @@ -13,11 +13,6 @@ ;; echo '1' | java -agentlib:native-image-agent=config-output-dir=/tmp -jar target/babashka-xxx-standalone.jar '...' ;; with the java provided by GraalVM. -(defn read-edn [s] - (edn/read-string - {:readers *data-readers*} - s)) - (defn- parse-opts [options] (let [opts (loop [options options opts-map {}] @@ -25,6 +20,9 @@ (case opt ("--version") {:version true} ("--help") {:help? true} + ("--stream") (recur (rest options) + (assoc opts-map + :stream? true)) ("-i") (recur (rest options) (assoc opts-map :raw-in true)) @@ -55,7 +53,7 @@ (defn print-version [] (println (str "babashka v"(str/trim (slurp (io/resource "BABASHKA_VERSION")))))) -(def usage-string "Usage: bb [ --help ] | [ --version ] | ( [ -i ] [ -o ] | [ -io ] ) ( expression | -f )") +(def usage-string "Usage: bb [ --help ] | [ --version ] | ( [ -i ] [ -o ] | [ -io ] ) [ --stream ] ( expression | -f )") (defn print-usage [] (println usage-string)) @@ -73,7 +71,8 @@ -i: read shell input into a list of strings instead of reading EDN. -o: write shell output instead of EDN. -io: combination of -i and -o. - --file or -f: read expression from file instead of argument + --stream: stream over lines or EDN values from stdin. Combined with -i *in* becomes a single line per iteration. + --file or -f: read expressions from file instead of argument wrapped in an implicit do. ")) (defn read-file [file] @@ -118,16 +117,20 @@ 'System/getProperties get-properties 'System/exit exit}) +(defn read-edn [] + (edn/read {;;:readers *data-readers* + :eof ::EOF} *in*)) + (defn main [& args] #_(binding [*out* *err*] - (prn ">> args" args)) + (prn ">> args" args)) (or (let [{:keys [:version :raw-in :raw-out :println? :help? :file :command-line-args - :expression] :as _opts} (parse-opts args)] + :expression :stream?] :as _opts} (parse-opts args)] #_(binding [*out* *err*] - (prn ">>" _opts)) + (prn ">>" _opts)) (second (cond version [(print-version) 0] @@ -135,25 +138,36 @@ [(print-help) 0] :else (try - [(do (when-not (or expression file) - (throw (Exception. "Missing expression."))) - (let [expr (if file (read-file file) expression) - in (delay (let [in (slurp *in*)] - (if raw-in - (parse-shell-string in) - (read-edn in)))) - res (sci/eval-string - expr - {:bindings (assoc bindings - (with-meta '*in* - {:sci/deref! true}) in - '*command-line-args* command-line-args)})] - (if raw-out - (if (coll? res) - (doseq [l res] - (println l)) - (println res)) - ((if println? println? prn) res)))) 0] + (let [expr (if file (read-file file) expression) + read-next #(if stream? + (if raw-in (or (read-line) ::EOF) + (read-edn)) + (delay (let [in (slurp *in*)] + (if raw-in + (parse-shell-string in) + (edn/read-string in)))))] + (loop [in (read-next)] + (if (identical? ::EOF in) + [nil 0] ;; done streaming + (let [res [(do (when-not (or expression file) + (throw (Exception. "Missing expression."))) + (let [res (sci/eval-string + expr + {:bindings (assoc bindings + (with-meta '*in* + (when-not stream? {:sci/deref! true})) in + #_(with-meta 'bb/*in* + {:sci/deref! true}) #_do-in + '*command-line-args* command-line-args)})] + (if raw-out + (if (coll? res) + (doseq [l res] + (println l)) + (println res)) + ((if println? println? prn) res)))) 0]] + (if stream? + (recur (read-next)) + res))))) (catch Exception e (binding [*out* *err*] (when-let [msg (or (:stderr (ex-data e)) diff --git a/test/babashka/main_test.clj b/test/babashka/main_test.clj index 6ff5fcdb..a66aca23 100644 --- a/test/babashka/main_test.clj +++ b/test/babashka/main_test.clj @@ -70,8 +70,20 @@ (doseq [s res] (is (not-empty s))))) -(deftest malformed-command-line-args +(deftest malformed-command-line-args-test (is (thrown-with-msg? Exception #"File does not exist: non-existing\n" (bb nil "-f" "non-existing"))) (is (thrown-with-msg? Exception #"Missing expression.\n" (bb nil)))) + +#_(deftest raw-in-test + (is (= "[1 2 3\n4 5 6 [\"1 2 3\" \"4 5 6\"]]" + (bb "1 2 3\n4 5 6" "-i" "(format \"[%s %s]\" bb/*in* *in*)'")))) + +(deftest stream-test + (is (= "2\n3\n4\n" (test-utils/bb "1 2 3" "--stream" "(inc *in*)"))) + (is (= "2\n3\n4\n" (test-utils/bb "{:x 2} {:x 3} {:x 4}" "--stream" "(:x *in*)"))) + (let [x "foo\n\bar\n"] + (is (= x (test-utils/bb x "--stream" "-io" "*in*")))) + (let [x "f\n\b\n"] + (is (= x (test-utils/bb x "--stream" "-io" "(subs *in* 0 1)")))))