[#14] implement --stream option

README.md

@@ -75,7 +75,7 @@ You may also download a binary from [Github](https://github.com/borkdude/babashk
 ## Usage
 
 ``` shellsession
-bb [ --help ] | [ --version ] | ( [ -i ] [ -o ] | [ -io ] ) ( expression | -f <file> )
+bb [ --help ] | [ --version ] | ( [ -i ] [ -o ] | [ -io ] ) [ --stream ] ( expression | -f <file> )
 ```
 
 Type `bb --help` to see a full explanation of the options.
@@ -84,7 +84,8 @@ The input is read as EDN by default. If the `-i` flag is provided, then the
 input is read as a string which is then split on newlines. The output is printed
 as EDN by default, unless the `-o` flag is provided, then the output is turned
 into shell-scripting friendly output. To combine `-i` and `-o` you can use
-`-io`.
+`-io`. When using the `--stream` option the expression is executed for every
+line or EDN value from stdin.
 
 The `clojure.core` functions are accessible without a namespace alias.
 
@@ -102,7 +103,8 @@ From Java the following is available:
 
 Special vars:
 
-- `*in*`: contains the input read from stdin
+- `*in*`: contains the input read from stdin (EDN by default, multiple lines with the `-i` option)
+<!-- - `bb/*in*`: the unprocessed input from stdin -->
 - `*command-line-args*`: contain the command line args
 
 Examples:

script/compile

@@ -11,6 +11,8 @@ fi
 
 BABASHKA_VERSION=$(cat resources/BABASHKA_VERSION)
 
+# We also need to AOT sci, else something didn't work in the Mac build on CircleCI
+# See https://github.com/oracle/graal/issues/1613
 ( cd /tmp;  git clone https://github.com/borkdude/sci 2> /dev/null || true )
 mkdir -p src/sci
 cp -R /tmp/sci/src/* src

src/babashka/main.clj

@@ -13,11 +13,6 @@
 ;; echo '1' | java -agentlib:native-image-agent=config-output-dir=/tmp -jar target/babashka-xxx-standalone.jar '...'
 ;; with the java provided by GraalVM.
 
-(defn read-edn [s]
-  (edn/read-string
-   {:readers *data-readers*}
-   s))
-
 (defn- parse-opts [options]
   (let [opts (loop [options options
                     opts-map {}]
@@ -25,6 +20,9 @@
                  (case opt
                    ("--version") {:version true}
                    ("--help") {:help? true}
+                   ("--stream") (recur (rest options)
+                                       (assoc opts-map
+                                              :stream? true))
                    ("-i") (recur (rest options)
                                  (assoc opts-map
                                         :raw-in true))
@@ -55,7 +53,7 @@
 (defn print-version []
   (println (str "babashka v"(str/trim (slurp (io/resource "BABASHKA_VERSION"))))))
 
-(def usage-string "Usage: bb [ --help ] | [ --version ] | ( [ -i ] [ -o ] | [ -io ] ) ( expression | -f <file> )")
+(def usage-string "Usage: bb [ --help ] | [ --version ] | ( [ -i ] [ -o ] | [ -io ] ) [ --stream ] ( expression | -f <file> )")
 (defn print-usage []
   (println usage-string))
 
@@ -73,7 +71,8 @@
   -i: read shell input into a list of strings instead of reading EDN.
   -o: write shell output instead of EDN.
   -io: combination of -i and -o.
-  --file or -f: read expression from file instead of argument
+  --stream: stream over lines or EDN values from stdin. Combined with -i *in* becomes a single line per iteration.
+  --file or -f: read expressions from file instead of argument wrapped in an implicit do.
 "))
 
 (defn read-file [file]
@@ -118,16 +117,20 @@
    'System/getProperties get-properties
    'System/exit exit})
 
+(defn read-edn []
+  (edn/read {;;:readers *data-readers*
+             :eof ::EOF} *in*))
+
 (defn main
   [& args]
   #_(binding [*out* *err*]
-    (prn ">> args" args))
+      (prn ">> args" args))
   (or
    (let [{:keys [:version :raw-in :raw-out :println?
                  :help? :file :command-line-args
-                 :expression] :as _opts} (parse-opts args)]
+                 :expression :stream?] :as _opts} (parse-opts args)]
      #_(binding [*out* *err*]
-       (prn ">>" _opts))
+         (prn ">>" _opts))
      (second
       (cond version
             [(print-version) 0]
@@ -135,25 +138,36 @@
             [(print-help) 0]
             :else
             (try
-              [(do (when-not (or expression file)
-                     (throw (Exception. "Missing expression.")))
-                 (let [expr (if file (read-file file) expression)
-                       in (delay (let [in (slurp *in*)]
-                                   (if raw-in
-                                     (parse-shell-string in)
-                                     (read-edn in))))
-                       res (sci/eval-string
-                            expr
-                            {:bindings (assoc bindings
-                                              (with-meta '*in*
-                                                {:sci/deref! true}) in
-                                              '*command-line-args* command-line-args)})]
-                   (if raw-out
-                     (if (coll? res)
-                       (doseq [l res]
-                         (println l))
-                       (println res))
-                     ((if println? println? prn) res)))) 0]
+              (let [expr (if file (read-file file) expression)
+                    read-next #(if stream?
+                                 (if raw-in (or (read-line) ::EOF)
+                                     (read-edn))
+                                 (delay (let [in (slurp *in*)]
+                                          (if raw-in
+                                            (parse-shell-string in)
+                                            (edn/read-string in)))))]
+                (loop [in (read-next)]
+                  (if (identical? ::EOF in)
+                    [nil 0] ;; done streaming
+                    (let [res [(do (when-not (or expression file)
+                                     (throw (Exception. "Missing expression.")))
+                                   (let [res (sci/eval-string
+                                              expr
+                                              {:bindings (assoc bindings
+                                                                (with-meta '*in*
+                                                                  (when-not stream? {:sci/deref! true})) in
+                                                                #_(with-meta 'bb/*in*
+                                                                    {:sci/deref! true}) #_do-in
+                                                                '*command-line-args* command-line-args)})]
+                                     (if raw-out
+                                       (if (coll? res)
+                                         (doseq [l res]
+                                           (println l))
+                                         (println res))
+                                       ((if println? println? prn) res)))) 0]]
+                      (if stream?
+                        (recur (read-next))
+                        res)))))
               (catch Exception e
                 (binding [*out* *err*]
                   (when-let [msg (or (:stderr (ex-data e))

test/babashka/main_test.clj

@@ -70,8 +70,20 @@
     (doseq [s res]
       (is (not-empty s)))))
 
-(deftest malformed-command-line-args
+(deftest malformed-command-line-args-test
   (is (thrown-with-msg? Exception #"File does not exist: non-existing\n"
                         (bb nil  "-f" "non-existing")))
   (is (thrown-with-msg? Exception #"Missing expression.\n"
                         (bb nil))))
+
+#_(deftest raw-in-test
+  (is (= "[1 2 3\n4 5 6 [\"1 2 3\" \"4 5 6\"]]"
+         (bb "1 2 3\n4 5 6" "-i" "(format \"[%s %s]\" bb/*in* *in*)'"))))
+
+(deftest stream-test
+  (is (= "2\n3\n4\n" (test-utils/bb "1 2 3" "--stream" "(inc *in*)")))
+  (is (= "2\n3\n4\n" (test-utils/bb "{:x 2} {:x 3} {:x 4}" "--stream" "(:x *in*)")))
+  (let [x "foo\n\bar\n"]
+    (is (= x (test-utils/bb x "--stream" "-io" "*in*"))))
+  (let [x "f\n\b\n"]
+    (is (= x (test-utils/bb x "--stream" "-io" "(subs *in* 0 1)")))))