[#48, #49, #50] add conch, Thread/sleep, wait-for-it, future and atom (#51)

.circleci/config.yml

@@ -18,7 +18,7 @@ jobs:
           name: "Pull Submodules"
           command: |
             git submodule init
-            git submodule update --remote
+            git submodule update
       - restore_cache:
           keys:
             - v1-dependencies-{{ checksum "project.clj" }}
@@ -77,7 +77,7 @@ jobs:
           name: "Pull Submodules"
           command: |
             git submodule init
-            git submodule update --remote
+            git submodule update
       - restore_cache:
           keys:
             - linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
@@ -146,7 +146,7 @@ jobs:
           name: "Pull Submodules"
           command: |
             git submodule init
-            git submodule update --remote
+            git submodule update
       - restore_cache:
           keys:
             - mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }}
@@ -208,7 +208,7 @@ jobs:
           name: "Pull Submodules"
           command: |
             git submodule init
-            git submodule update --remote
+            git submodule update
       - restore_cache:
           keys:
           - v1-dependencies-{{ checksum "project.clj" }}

README.md

@@ -115,7 +115,7 @@ Options:
 
 The `clojure.core` functions are accessible without a namespace alias.
 
-The following Clojure namespaces are required by default and only available
+The following namespaces are required by default and only available
 through the aliases. If not all vars are available, they are enumerated
 explicitly.
 
@@ -127,16 +127,19 @@ explicitly.
   - `sh`
 - `clojure.java.io` aliased as `io`:
   - `as-relative-path`, `copy`, `delete-file`, `file`
+- [`me.raynes.conch.low-level`](https://github.com/clj-commons/conch#low-level-usage)
+  aliased as `conch`
 
 From Java the following is available:
 
-- `System`: `exit`, `getProperty`, `setProperty`, `getProperties`, `getenv`
 - `File`: `.canRead`, `.canWrite`, `.delete`, `.deleteOnExit`, `.exists`,
   `.getAbsoluteFile`, `.getCanonicalFile`, `.getCanonicalPath`, `.getName`,
   `.getParent`, `.getParentFile`, `.getPath`, `.isAbsolute`, `.isDirectory`,
   `.isFile`, `.isHidden`, `.lastModified`, `.length`, `.list`, `.listFiles`,
   `.mkdir`, `.mkdirs`, `.renameTo`, `.setLastModified`, `.setReadOnly`,
   `.setReadable`, `.toPath`, `.toURI`.
+- `System`: `exit`, `getProperty`, `setProperty`, `getProperties`, `getenv`
+- `Thread`: `sleep`
 
 Special vars:
 
@@ -144,7 +147,20 @@ Special vars:
 text with the `-i` option, or multiple EDN values with the `-I` option.
 - `*command-line-args*`: contain the command line args
 
-Examples:
+Additionally, babashka adds the following functions:
+
+- `net/wait-for-it`. Usage:
+
+``` clojure
+(net/wait-for-it "localhost" 8080)
+(net/wait-for-it "localhost" 8080 {:timeout 1000 :pause 1000)
+```
+
+Waits for TCP connection to be available on host and port. Options map supports
+  `:timeout` and `:pause`. If `:timeout` is provided and reached, exception will
+  be thrown. The `:pause` option determines the time waited between retries.
+
+## Examples
 
 ``` shellsession
 $ ls | bb -i '*in*'
@@ -275,6 +291,19 @@ $
 A socket REPL client for Emacs is
 [inf-clojure](https://github.com/clojure-emacs/inf-clojure).
 
+## Spawning and killing a process
+
+You may use the `conch` namespace for this. It maps to
+[`me.raynes.conch.low-level`](https://github.com/clj-commons/conch#low-level-usage).
+
+Example:
+
+``` clojure
+$ bb '
+(def ws (conch/proc "python" "-m" "SimpleHTTPServer" "1777"))
+(net/wait-for-it "localhost" 1777) (conch/destroy ws)'
+```
+
 ## Enabling SSL
 
 If you want to be able to use SSL to e.g. run `(slurp
@@ -381,5 +410,9 @@ bb '(some #(re-find #".*linux.*" (:browser_download_url %)) *in*)'
 
 Copyright © 2019 Michiel Borkent
 
-Distributed under the EPL License. This project contains modified Clojure code
+Distributed under the EPL License. See LICENSE.
-which is licensed under the same EPL License. See LICENSE.
+
+This project contains code from:
+- Clojure, which is licensed under the same EPL License.
+- [conch](https://github.com/clj-commons/conch), which is licensed under the
+same EPL License.

project.clj

@@ -7,7 +7,7 @@
         :url "https://github.com/borkdude/babashka"}
   :license {:name "Eclipse Public License 1.0"
             :url "http://opensource.org/licenses/eclipse-1.0.php"}
-  :source-paths ["src" "sci/src" "sci/inlined"]
+  :source-paths ["src" "sci/src" "sci/inlined" "conch/src"]
   :resource-paths ["resources" "sci/resources"]
   :dependencies [[org.clojure/clojure "1.10.1"]]
   :profiles {:test {:dependencies [[clj-commons/conch "0.9.2"]]}

sci

@@ -1 +1 @@
-Subproject commit 4ab01bd92cc87511f478c861c84b40c6e208eeae
+Subproject commit 8715ef2de4320a75436cf93d624fb99e2d6a3600

src/babashka/impl/File.clj

@@ -57,7 +57,7 @@
 (gen-wrapper-fn toPath)
 (gen-wrapper-fn toURI)
 
-(def bindings
+(def file-bindings
   (reduce (fn [acc [k v]]
             (if (-> v meta :bb/export)
               (assoc acc (symbol (str "." k))

src/babashka/impl/System.clj

@@ -0,0 +1,28 @@
+(ns babashka.impl.System
+  {:no-doc true})
+
+(defn get-env
+  ([] (System/getenv))
+  ([s] (System/getenv s)))
+
+(defn get-property
+  ([s]
+   (System/getProperty s))
+  ([s d]
+   (System/getProperty s d)))
+
+(defn set-property [k v]
+  (System/setProperty k v))
+
+(defn get-properties []
+  (System/getProperties))
+
+(defn exit [n]
+  (throw (ex-info "" {:bb/exit-code n})))
+
+(def system-bindings
+  {'System/getenv get-env
+   'System/getProperty get-property
+   'System/setProperty set-property
+   'System/getProperties get-properties
+   'System/exit exit})

src/babashka/impl/Thread.clj

@@ -0,0 +1,9 @@
+(ns babashka.impl.Thread
+  {:no-doc true})
+
+(defn sleep
+  ([millis] (Thread/sleep millis))
+  ([millis nanos] (Thread/sleep millis nanos)))
+
+(def thread-bindings
+  {'Thread/sleep sleep})

src/babashka/impl/clojure/core.clj

@@ -0,0 +1,26 @@
+(ns babashka.impl.clojure.core
+  {:no-doc true}
+  (:refer-clojure :exclude [future]))
+
+(defn future
+  [& body]
+  `(~'future-call (fn [] ~@body)))
+
+(def core-bindings
+  {;; atoms
+   'atom atom
+   'swap! swap!
+   'reset! reset!
+   'add-watch add-watch
+
+   'run! run!
+   'slurp slurp
+   'spit spit
+   'pmap pmap
+   'print print
+   'pr-str pr-str
+   'prn prn
+   'println println
+   'future-call future-call
+   'future (with-meta future {:sci/macro true})
+   'deref deref})

src/babashka/impl/clojure/stacktrace.clj

@@ -0,0 +1,88 @@
+;;   Copyright (c) Rich Hickey. All rights reserved.
+;;   The use and distribution terms for this software are covered by the
+;;   Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+;;   which can be found in the file epl-v10.html at the root of this distribution.
+;;   By using this software in any fashion, you are agreeing to be bound by
+;;   the terms of this license.
+;;   You must not remove this notice, or any other, from this software.
+
+;;; stacktrace.clj: print Clojure-centric stack traces
+
+;; by Stuart Sierra
+;; January 6, 2009
+
+(ns ^{:doc "Print stack traces oriented towards Clojure, not Java."
+      :author "Stuart Sierra"
+      :no-doc true}
+    babashka.impl.clojure.stacktrace)
+
+(set! *warn-on-reflection* true)
+
+(defn root-cause
+  "Returns the last 'cause' Throwable in a chain of Throwables."
+  {:added "1.1"}
+  [^Throwable tr]
+  (if-let [cause (.getCause tr)]
+    (recur cause)
+    tr))
+
+(defn print-trace-element
+  "Prints a Clojure-oriented view of one element in a stack trace."
+  {:added "1.1"}
+  [^StackTraceElement e]
+  (let [class (.getClassName e)
+        method (.getMethodName e)]
+    (let [match (re-matches #"^([A-Za-z0-9_.-]+)\$(\w+)__\d+$" (str class))]
+      (if (and match (= "invoke" method))
+        (apply printf "%s/%s" (rest match))
+        (printf "%s.%s" class method))))
+  (printf " (%s:%d)" (or (.getFileName e) "") (.getLineNumber e)))
+
+(defn print-throwable
+  "Prints the class and message of a Throwable. Prints the ex-data map
+  if present."
+  {:added "1.1"}
+  [^Throwable tr]
+  (printf "%s: %s" (.getName (class tr)) (.getMessage tr))
+  (when-let [info (ex-data tr)]
+    (newline)
+    (pr info)))
+
+(defn print-stack-trace
+  "Prints a Clojure-oriented stack trace of tr, a Throwable.
+  Prints a maximum of n stack frames (default: unlimited).
+  Does not print chained exceptions (causes)."
+  {:added "1.1"}
+  ([tr] (print-stack-trace tr nil))
+  ([^Throwable tr n]
+   (let [st (.getStackTrace tr)]
+     (print-throwable tr)
+     (newline)
+     (print " at ")
+     (if-let [e (first st)]
+       (print-trace-element e)
+       (print "[empty stack trace]"))
+     (newline)
+     (doseq [e (if (nil? n)
+                 (rest st)
+                 (take (dec n) (rest st)))]
+       (print "    ")
+       (print-trace-element e)
+       (newline)))))
+
+(defn print-cause-trace
+  "Like print-stack-trace but prints chained exceptions (causes)."
+  {:added "1.1"}
+  ([tr] (print-cause-trace tr nil))
+  ([^Throwable tr n]
+   (print-stack-trace tr n)
+   (when-let [cause (.getCause tr)]
+     (print "Caused by: " )
+     (recur cause n))))
+
+(defn e
+  "REPL utility.  Prints a brief stack trace for the root cause of the
+  most recent exception."
+  {:added "1.1"}
+  []
+  (print-stack-trace (root-cause *e) 8))

src/babashka/impl/conch.clj

@@ -0,0 +1,18 @@
+(ns babashka.impl.conch
+  {:no-doc true}
+  (:require
+   [babashka.impl.me.raynes.conch.low-level :as ll]))
+
+(def conch-bindings
+  {;; low level API
+   'conch/proc ll/proc
+   'conch/destroy ll/destroy
+   'conch/exit-code ll/exit-code
+   'conch/flush ll/flush
+   'conch/done ll/done
+   'conch/stream-to ll/stream-to
+   'conch/feed-from ll/feed-from
+   'conch/stream-to-string ll/stream-to-string
+   'conch/stream-to-out ll/stream-to-out
+   'conch/feed-from-string ll/feed-from-string
+   'conch/read-line ll/read-line})

src/babashka/impl/me/raynes/conch/low_level.clj

@@ -0,0 +1,126 @@
+;; From https://github.com/clj-commons/conch
+
+(ns babashka.impl.me.raynes.conch.low-level
+  "A simple but flexible library for shelling out from Clojure."
+  {:no-doc true}
+  (:refer-clojure :exclude [flush read-line])
+  (:require [clojure.java.io :as io])
+  (:import [java.util.concurrent TimeUnit TimeoutException]
+           [java.io InputStream OutputStream]))
+
+(set! *warn-on-reflection* true)
+
+(defn proc
+  "Spin off another process. Returns the process's input stream,
+  output stream, and err stream as a map of :in, :out, and :err keys
+  If passed the optional :dir and/or :env keyword options, the dir
+  and enviroment will be set to what you specify. If you pass
+  :verbose and it is true, commands will be printed. If it is set to
+  :very, environment variables passed, dir, and the command will be
+  printed. If passed the :clear-env keyword option, then the process
+  will not inherit its environment from its parent process."
+  [& args]
+  (let [[cmd args] (split-with (complement keyword?) args)
+        args (apply hash-map args)
+        builder (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String cmd))
+        env (.environment builder)]
+    (when (:clear-env args)
+      (.clear env))
+    (doseq [[k v] (:env args)]
+      (.put env k v))
+    (when-let [dir (:dir args)]
+      (.directory builder (io/file dir)))
+    (when (:verbose args) (apply println cmd))
+    (when (= :very (:verbose args))
+      (when-let [env (:env args)] (prn env))
+      (when-let [dir (:dir args)] (prn dir)))
+    (when (:redirect-err args)
+      (.redirectErrorStream builder true))
+    (let [process (.start builder)]
+      {:out (.getInputStream process)
+       :in  (.getOutputStream process)
+       :err (.getErrorStream process)
+       :process process})))
+
+(defn destroy
+  "Destroy a process."
+  [process]
+  (.destroy ^Process (:process process)))
+
+;; .waitFor returns the exit code. This makes this function useful for
+;; both getting an exit code and stopping the thread until a process
+;; terminates.
+(defn exit-code
+  "Waits for the process to terminate (blocking the thread) and returns
+   the exit code. If timeout is passed, it is assumed to be milliseconds
+   to wait for the process to exit. If it does not exit in time, it is
+   killed (with or without fire)."
+  ([process] (.waitFor ^Process (:process process)))
+  ([process timeout]
+   (try
+     (let [^java.util.concurrent.Future fut
+           (future (.waitFor ^Process (:process process)))]
+       (.get fut timeout TimeUnit/MILLISECONDS))
+     (catch Exception e
+       (if (or (instance? TimeoutException e)
+               (instance? TimeoutException (.getCause e)))
+         (do (destroy process)
+             :timeout)
+         (throw e))))))
+
+(defn flush
+  "Flush the output stream of a process."
+  [process]
+  (let [^OutputStream in (:in process)]
+    (.flush in)))
+
+(defn done
+  "Close the process's output stream (sending EOF)."
+  [proc]
+  (let [^OutputStream in (:in proc)]
+    (.close in)))
+
+(defn stream-to
+  "Stream :out or :err from a process to an ouput stream.
+  Options passed are fed to clojure.java.io/copy. They are :encoding to
+  set the encoding and :buffer-size to set the size of the buffer.
+  :encoding defaults to UTF-8 and :buffer-size to 1024."
+  [process from to & args]
+  (apply io/copy (process from) to args))
+
+(defn feed-from
+  "Feed to a process's input stream with optional. Options passed are
+  fed to clojure.java.io/copy. They are :encoding to set the encoding
+  and :buffer-size to set the size of the buffer. :encoding defaults to
+  UTF-8 and :buffer-size to 1024. If :flush is specified and is false,
+  the process will be flushed after writing."
+  [process from & {flush? :flush :or {flush? true} :as all}]
+  (apply io/copy from (:in process) all)
+  (when flush? (flush process)))
+
+(defn stream-to-string
+  "Streams the output of the process to a string and returns it."
+  [process from & args]
+  (with-open [writer (java.io.StringWriter.)]
+    (apply stream-to process from writer args)
+    (str writer)))
+
+;; The writer that Clojure wraps System/out in for *out* seems to buffer
+;; things instead of writing them immediately. This wont work if you
+;; really want to stream stuff, so we'll just skip it and throw our data
+;; directly at System/out.
+(defn stream-to-out
+  "Streams the output of the process to System/out"
+  [process from & args]
+  (apply stream-to process from (System/out) args))
+
+(defn feed-from-string
+  "Feed the process some data from a string."
+  [process s & args]
+  (apply feed-from process (java.io.StringReader. s) args))
+
+(defn read-line
+  "Read a line from a process' :out or :err."
+  [process from]
+  (binding [*in* (io/reader (from process))]
+    (clojure.core/read-line)))

src/babashka/main.clj

@@ -1,13 +1,19 @@
 (ns babashka.main
   {:no-doc true}
   (:require
-   [babashka.impl.File :as File]
+   [babashka.impl.File :refer [file-bindings]]
+   [babashka.impl.System :refer [system-bindings]]
+   [babashka.impl.Thread :refer [thread-bindings]]
+   [babashka.impl.clojure.core :refer [core-bindings]]
+   [babashka.impl.clojure.stacktrace :refer [print-stack-trace]]
+   [babashka.impl.conch :refer [conch-bindings]]
    [babashka.impl.pipe-signal-handler :refer [handle-pipe! pipe-signal-received?]]
+   [babashka.impl.socket-repl :as socket-repl]
+   [babashka.net :as net]
    [clojure.edn :as edn]
    [clojure.java.io :as io]
    [clojure.java.shell :as shell]
    [clojure.string :as str]
-   [babashka.impl.socket-repl :as socket-repl]
    [sci.core :as sci])
   (:import [sun.misc Signal]
            [sun.misc SignalHandler])
@@ -120,55 +126,22 @@
         (str/replace x #"^#!.*" ""))
       (throw (Exception. (str "File does not exist: " file))))))
 
-(defn get-env
-  ([] (System/getenv))
-  ([s] (System/getenv s)))
-
-(defn get-property
-  ([s]
-   (System/getProperty s))
-  ([s d]
-   (System/getProperty s d)))
-
-(defn set-property [k v]
-  (System/setProperty k v))
-
-(defn get-properties []
-  (System/getProperties))
-
-(defn exit [n]
-  (throw (ex-info "" {:bb/exit-code n})))
-
 (def bindings
-  (merge {'run! run!
+  (merge {'shell/sh shell/sh
-          'shell/sh shell/sh
-          'csh shell/sh ;; backwards compatibility, deprecated
           'namespace namespace
-          'slurp slurp
-          'spit spit
-          'pmap pmap
-          'print print
-          'pr-str pr-str
-          'prn prn
-          'println println
-
           ;; clojure.java.io
           'io/as-relative-path io/as-relative-path
           'io/copy io/copy
           'io/delete-file io/delete-file
           'io/file io/file
-          ;; '.canRead File/canRead
+          'io/reader io/reader
-          ;; '.canWrite File/canWrite
-          ;; '.exists File/exists
-          ;; '.delete File/delete
-
           'edn/read-string edn/read-string
-          'System/getenv get-env
+          'net/wait-for-it net/wait-for-it}
-          'System/getProperty get-property
+         core-bindings
-          'System/setProperty set-property
+         system-bindings
-          'System/getProperties get-properties
+         file-bindings
-          'System/exit exit}
+         thread-bindings
-         File/bindings))
+         conch-bindings))
 
 (defn read-edn []
   (edn/read {;;:readers *data-readers*
@@ -191,7 +164,7 @@
   [& args]
   (handle-pipe!)
   #_(binding [*out* *err*]
-      (prn ">> args" args))
+    (prn "M" (meta (get bindings 'future))))
   (let [t0 (System/currentTimeMillis)
         {:keys [:version :shell-in :edn-in :shell-out :edn-out
                 :help? :file :command-line-args
@@ -236,25 +209,25 @@
                           (let [res [(do (when-not (or expression file)
                                            (throw (Exception. (str args  "Babashka expected an expression. Type --help to print help."))))
                                          (let [res (sci/eval-string expr ctx)]
-                                           (if-let [pr-f (cond shell-out println
+                                           (if (some? res)
-                                                               edn-out prn)]
+                                             (if-let [pr-f (cond shell-out println
-                                             (if (coll? res)
+                                                                 edn-out prn)]
-                                               (doseq [l res
+                                               (if (coll? res)
-                                                       :while (not (pipe-signal-received?))]
+                                                 (doseq [l res
-                                                 (pr-f l))
+                                                         :while (not (pipe-signal-received?))]
-                                               (pr-f res))
+                                                   (pr-f l))
-                                             (prn res)))) 0]]
+                                                 (pr-f res))
+                                               (prn res))))) 0]]
                             (if stream?
                               (recur (read-next *in*))
                               res))))))
-                  (catch Exception e
+                  (catch Throwable e
                     (binding [*out* *err*]
                       (let [d (ex-data e)
                             exit-code (:bb/exit-code d)]
                         (if exit-code [nil exit-code]
-                            (do (when-let [msg (or (:stderr d )
+                            (do (print-stack-trace e)
-                                                   (.getMessage e))]
+                                (flush)
-                                  (println (str/trim msg)))
                                 [nil 1]))))))))
          1)
         t1 (System/currentTimeMillis)]

src/babashka/net.clj

@@ -0,0 +1,37 @@
+(ns babashka.net
+  (:import [java.net Socket ConnectException]))
+
+(set! *warn-on-reflection* true)
+
+(defn wait-for-it
+  "Waits for TCP connection to be available on host and port. Options map
+  supports `:timeout` and `:pause`. If `:timeout` is provided and reached,
+  exception will be thrown. The `:pause` option determines the time waited
+  between retries."
+  ([host port]
+   (wait-for-it host port nil))
+  ([^String host ^long port {:keys [:timeout :pause] :as opts}]
+   (let [opts (merge {:host host
+                      :port port}
+                     opts)
+         t0 (System/currentTimeMillis)]
+     (loop []
+       (let [v (try (Socket. host port)
+                    (- (System/currentTimeMillis) t0)
+                    (catch ConnectException _e
+                      (let [took (- (System/currentTimeMillis) t0)]
+                        (if (and timeout (>= took timeout))
+                          (throw (ex-info
+                                  (format "timeout while waiting for %s:%s" host port)
+                                  (assoc opts :took took)))
+                          :wait-for-it.impl/try-again))))]
+         (if (identical? :wait-for-it.impl/try-again v)
+           (do (Thread/sleep (or pause 100))
+               (recur))
+           (assoc opts :took v)))))))
+
+(comment
+  (wait-for-it "localhost" 80)
+  (wait-for-it "localhost" 80 {:timeout 1000})
+  (wait-for-it "google.com" 80)
+  )

test/babashka/impl/socket_repl_test.clj

@@ -43,7 +43,7 @@
                                       "echo \"(inc 1336)\" | nc -q 1 127.0.0.1 1666")))
                           "1337\nbb=> ")))
     (testing "*in*"
-      (is (str/includes? (socket-command '*in*)
+      (is (str/includes? (socket-command "*in*")
                          "[1 2 3]")))
     (testing "*command-line-args*"
       (is (str/includes? (socket-command '*command-line-args*)

test/babashka/main_test.clj

@@ -80,7 +80,7 @@
 
 (deftest malformed-command-line-args-test
   (is (thrown-with-msg? Exception #"File does not exist: non-existing\n"
-                        (bb nil  "-f" "non-existing")))
+                        (bb nil "-f" "non-existing")))
   (is (thrown-with-msg? Exception #"expression"
                         (bb nil))))
 
@@ -136,3 +136,19 @@
     (let [out (:out (sh "bash" "-c" "yes | ./bb -i '(take 2 *in*)'"))
           out (edn/read-string out)]
       (is (= '("y" "y") out)))))
+
+(deftest future-test
+  (is (= 6 (bb nil "@(future (+ 1 2 3))"))))
+
+(deftest conch-test
+  (is (str/includes? (bb nil "(->> (conch/proc \"ls\") (conch/stream-to-string :out))")
+                     "LICENSE")))
+
+(deftest wait-for-it-test
+  (is (thrown-with-msg?
+       Exception
+       #"timeout"
+       (bb nil "(def web-server (conch/proc \"python\" \"-m\" \"SimpleHTTPServer\" \"7171\"))
+                (net/wait-for-it \"127.0.0.1\" 7171)
+                (conch/destroy web-server)
+                (net/wait-for-it \"localhost\" 7172 {:timeout 50})"))))