diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b30d8fb..d17d6ee4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: name: "Pull Submodules" command: | git submodule init - git submodule update --remote + git submodule update - restore_cache: keys: - v1-dependencies-{{ checksum "project.clj" }} @@ -77,7 +77,7 @@ jobs: name: "Pull Submodules" command: | git submodule init - git submodule update --remote + git submodule update - restore_cache: keys: - linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} @@ -146,7 +146,7 @@ jobs: name: "Pull Submodules" command: | git submodule init - git submodule update --remote + git submodule update - restore_cache: keys: - mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} @@ -208,7 +208,7 @@ jobs: name: "Pull Submodules" command: | git submodule init - git submodule update --remote + git submodule update - restore_cache: keys: - v1-dependencies-{{ checksum "project.clj" }} diff --git a/README.md b/README.md index 80dd672c..281f2b99 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Options: The `clojure.core` functions are accessible without a namespace alias. -The following Clojure namespaces are required by default and only available +The following namespaces are required by default and only available through the aliases. If not all vars are available, they are enumerated explicitly. @@ -127,16 +127,19 @@ explicitly. - `sh` - `clojure.java.io` aliased as `io`: - `as-relative-path`, `copy`, `delete-file`, `file` +- [`me.raynes.conch.low-level`](https://github.com/clj-commons/conch#low-level-usage) + aliased as `conch` From Java the following is available: -- `System`: `exit`, `getProperty`, `setProperty`, `getProperties`, `getenv` - `File`: `.canRead`, `.canWrite`, `.delete`, `.deleteOnExit`, `.exists`, `.getAbsoluteFile`, `.getCanonicalFile`, `.getCanonicalPath`, `.getName`, `.getParent`, `.getParentFile`, `.getPath`, `.isAbsolute`, `.isDirectory`, `.isFile`, `.isHidden`, `.lastModified`, `.length`, `.list`, `.listFiles`, `.mkdir`, `.mkdirs`, `.renameTo`, `.setLastModified`, `.setReadOnly`, `.setReadable`, `.toPath`, `.toURI`. +- `System`: `exit`, `getProperty`, `setProperty`, `getProperties`, `getenv` +- `Thread`: `sleep` Special vars: @@ -144,7 +147,20 @@ Special vars: text with the `-i` option, or multiple EDN values with the `-I` option. - `*command-line-args*`: contain the command line args -Examples: +Additionally, babashka adds the following functions: + +- `net/wait-for-it`. Usage: + +``` clojure +(net/wait-for-it "localhost" 8080) +(net/wait-for-it "localhost" 8080 {:timeout 1000 :pause 1000) +``` + +Waits for TCP connection to be available on host and port. Options map supports + `:timeout` and `:pause`. If `:timeout` is provided and reached, exception will + be thrown. The `:pause` option determines the time waited between retries. + +## Examples ``` shellsession $ ls | bb -i '*in*' @@ -275,6 +291,19 @@ $ A socket REPL client for Emacs is [inf-clojure](https://github.com/clojure-emacs/inf-clojure). +## Spawning and killing a process + +You may use the `conch` namespace for this. It maps to +[`me.raynes.conch.low-level`](https://github.com/clj-commons/conch#low-level-usage). + +Example: + +``` clojure +$ bb ' +(def ws (conch/proc "python" "-m" "SimpleHTTPServer" "1777")) +(net/wait-for-it "localhost" 1777) (conch/destroy ws)' +``` + ## Enabling SSL If you want to be able to use SSL to e.g. run `(slurp @@ -381,5 +410,9 @@ bb '(some #(re-find #".*linux.*" (:browser_download_url %)) *in*)' Copyright © 2019 Michiel Borkent -Distributed under the EPL License. This project contains modified Clojure code -which is licensed under the same EPL License. See LICENSE. +Distributed under the EPL License. See LICENSE. + +This project contains code from: +- Clojure, which is licensed under the same EPL License. +- [conch](https://github.com/clj-commons/conch), which is licensed under the +same EPL License. diff --git a/project.clj b/project.clj index e51e7753..a457e952 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ :url "https://github.com/borkdude/babashka"} :license {:name "Eclipse Public License 1.0" :url "http://opensource.org/licenses/eclipse-1.0.php"} - :source-paths ["src" "sci/src" "sci/inlined"] + :source-paths ["src" "sci/src" "sci/inlined" "conch/src"] :resource-paths ["resources" "sci/resources"] :dependencies [[org.clojure/clojure "1.10.1"]] :profiles {:test {:dependencies [[clj-commons/conch "0.9.2"]]} diff --git a/sci b/sci index 4ab01bd9..8715ef2d 160000 --- a/sci +++ b/sci @@ -1 +1 @@ -Subproject commit 4ab01bd92cc87511f478c861c84b40c6e208eeae +Subproject commit 8715ef2de4320a75436cf93d624fb99e2d6a3600 diff --git a/src/babashka/impl/File.clj b/src/babashka/impl/File.clj index d00468b8..109c20e8 100644 --- a/src/babashka/impl/File.clj +++ b/src/babashka/impl/File.clj @@ -57,7 +57,7 @@ (gen-wrapper-fn toPath) (gen-wrapper-fn toURI) -(def bindings +(def file-bindings (reduce (fn [acc [k v]] (if (-> v meta :bb/export) (assoc acc (symbol (str "." k)) diff --git a/src/babashka/impl/System.clj b/src/babashka/impl/System.clj new file mode 100644 index 00000000..a937e990 --- /dev/null +++ b/src/babashka/impl/System.clj @@ -0,0 +1,28 @@ +(ns babashka.impl.System + {:no-doc true}) + +(defn get-env + ([] (System/getenv)) + ([s] (System/getenv s))) + +(defn get-property + ([s] + (System/getProperty s)) + ([s d] + (System/getProperty s d))) + +(defn set-property [k v] + (System/setProperty k v)) + +(defn get-properties [] + (System/getProperties)) + +(defn exit [n] + (throw (ex-info "" {:bb/exit-code n}))) + +(def system-bindings + {'System/getenv get-env + 'System/getProperty get-property + 'System/setProperty set-property + 'System/getProperties get-properties + 'System/exit exit}) diff --git a/src/babashka/impl/Thread.clj b/src/babashka/impl/Thread.clj new file mode 100644 index 00000000..d224d986 --- /dev/null +++ b/src/babashka/impl/Thread.clj @@ -0,0 +1,9 @@ +(ns babashka.impl.Thread + {:no-doc true}) + +(defn sleep + ([millis] (Thread/sleep millis)) + ([millis nanos] (Thread/sleep millis nanos))) + +(def thread-bindings + {'Thread/sleep sleep}) diff --git a/src/babashka/impl/clojure/core.clj b/src/babashka/impl/clojure/core.clj new file mode 100644 index 00000000..eb41f32c --- /dev/null +++ b/src/babashka/impl/clojure/core.clj @@ -0,0 +1,26 @@ +(ns babashka.impl.clojure.core + {:no-doc true} + (:refer-clojure :exclude [future])) + +(defn future + [& body] + `(~'future-call (fn [] ~@body))) + +(def core-bindings + {;; atoms + 'atom atom + 'swap! swap! + 'reset! reset! + 'add-watch add-watch + + 'run! run! + 'slurp slurp + 'spit spit + 'pmap pmap + 'print print + 'pr-str pr-str + 'prn prn + 'println println + 'future-call future-call + 'future (with-meta future {:sci/macro true}) + 'deref deref}) diff --git a/src/babashka/impl/clojure/stacktrace.clj b/src/babashka/impl/clojure/stacktrace.clj new file mode 100644 index 00000000..a220fd88 --- /dev/null +++ b/src/babashka/impl/clojure/stacktrace.clj @@ -0,0 +1,88 @@ +;; Copyright (c) Rich Hickey. All rights reserved. +;; The use and distribution terms for this software are covered by the +;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) +;; which can be found in the file epl-v10.html at the root of this distribution. +;; By using this software in any fashion, you are agreeing to be bound by +;; the terms of this license. +;; You must not remove this notice, or any other, from this software. + +;;; stacktrace.clj: print Clojure-centric stack traces + +;; by Stuart Sierra +;; January 6, 2009 + +(ns ^{:doc "Print stack traces oriented towards Clojure, not Java." + :author "Stuart Sierra" + :no-doc true} + babashka.impl.clojure.stacktrace) + +(set! *warn-on-reflection* true) + +(defn root-cause + "Returns the last 'cause' Throwable in a chain of Throwables." + {:added "1.1"} + [^Throwable tr] + (if-let [cause (.getCause tr)] + (recur cause) + tr)) + +(defn print-trace-element + "Prints a Clojure-oriented view of one element in a stack trace." + {:added "1.1"} + [^StackTraceElement e] + (let [class (.getClassName e) + method (.getMethodName e)] + (let [match (re-matches #"^([A-Za-z0-9_.-]+)\$(\w+)__\d+$" (str class))] + (if (and match (= "invoke" method)) + (apply printf "%s/%s" (rest match)) + (printf "%s.%s" class method)))) + (printf " (%s:%d)" (or (.getFileName e) "") (.getLineNumber e))) + +(defn print-throwable + "Prints the class and message of a Throwable. Prints the ex-data map + if present." + {:added "1.1"} + [^Throwable tr] + (printf "%s: %s" (.getName (class tr)) (.getMessage tr)) + (when-let [info (ex-data tr)] + (newline) + (pr info))) + +(defn print-stack-trace + "Prints a Clojure-oriented stack trace of tr, a Throwable. + Prints a maximum of n stack frames (default: unlimited). + Does not print chained exceptions (causes)." + {:added "1.1"} + ([tr] (print-stack-trace tr nil)) + ([^Throwable tr n] + (let [st (.getStackTrace tr)] + (print-throwable tr) + (newline) + (print " at ") + (if-let [e (first st)] + (print-trace-element e) + (print "[empty stack trace]")) + (newline) + (doseq [e (if (nil? n) + (rest st) + (take (dec n) (rest st)))] + (print " ") + (print-trace-element e) + (newline))))) + +(defn print-cause-trace + "Like print-stack-trace but prints chained exceptions (causes)." + {:added "1.1"} + ([tr] (print-cause-trace tr nil)) + ([^Throwable tr n] + (print-stack-trace tr n) + (when-let [cause (.getCause tr)] + (print "Caused by: " ) + (recur cause n)))) + +(defn e + "REPL utility. Prints a brief stack trace for the root cause of the + most recent exception." + {:added "1.1"} + [] + (print-stack-trace (root-cause *e) 8)) diff --git a/src/babashka/impl/conch.clj b/src/babashka/impl/conch.clj new file mode 100644 index 00000000..1597002d --- /dev/null +++ b/src/babashka/impl/conch.clj @@ -0,0 +1,18 @@ +(ns babashka.impl.conch + {:no-doc true} + (:require + [babashka.impl.me.raynes.conch.low-level :as ll])) + +(def conch-bindings + {;; low level API + 'conch/proc ll/proc + 'conch/destroy ll/destroy + 'conch/exit-code ll/exit-code + 'conch/flush ll/flush + 'conch/done ll/done + 'conch/stream-to ll/stream-to + 'conch/feed-from ll/feed-from + 'conch/stream-to-string ll/stream-to-string + 'conch/stream-to-out ll/stream-to-out + 'conch/feed-from-string ll/feed-from-string + 'conch/read-line ll/read-line}) diff --git a/src/babashka/impl/me/raynes/conch/low_level.clj b/src/babashka/impl/me/raynes/conch/low_level.clj new file mode 100644 index 00000000..0de31bdc --- /dev/null +++ b/src/babashka/impl/me/raynes/conch/low_level.clj @@ -0,0 +1,126 @@ +;; From https://github.com/clj-commons/conch + +(ns babashka.impl.me.raynes.conch.low-level + "A simple but flexible library for shelling out from Clojure." + {:no-doc true} + (:refer-clojure :exclude [flush read-line]) + (:require [clojure.java.io :as io]) + (:import [java.util.concurrent TimeUnit TimeoutException] + [java.io InputStream OutputStream])) + +(set! *warn-on-reflection* true) + +(defn proc + "Spin off another process. Returns the process's input stream, + output stream, and err stream as a map of :in, :out, and :err keys + If passed the optional :dir and/or :env keyword options, the dir + and enviroment will be set to what you specify. If you pass + :verbose and it is true, commands will be printed. If it is set to + :very, environment variables passed, dir, and the command will be + printed. If passed the :clear-env keyword option, then the process + will not inherit its environment from its parent process." + [& args] + (let [[cmd args] (split-with (complement keyword?) args) + args (apply hash-map args) + builder (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String cmd)) + env (.environment builder)] + (when (:clear-env args) + (.clear env)) + (doseq [[k v] (:env args)] + (.put env k v)) + (when-let [dir (:dir args)] + (.directory builder (io/file dir))) + (when (:verbose args) (apply println cmd)) + (when (= :very (:verbose args)) + (when-let [env (:env args)] (prn env)) + (when-let [dir (:dir args)] (prn dir))) + (when (:redirect-err args) + (.redirectErrorStream builder true)) + (let [process (.start builder)] + {:out (.getInputStream process) + :in (.getOutputStream process) + :err (.getErrorStream process) + :process process}))) + +(defn destroy + "Destroy a process." + [process] + (.destroy ^Process (:process process))) + +;; .waitFor returns the exit code. This makes this function useful for +;; both getting an exit code and stopping the thread until a process +;; terminates. +(defn exit-code + "Waits for the process to terminate (blocking the thread) and returns + the exit code. If timeout is passed, it is assumed to be milliseconds + to wait for the process to exit. If it does not exit in time, it is + killed (with or without fire)." + ([process] (.waitFor ^Process (:process process))) + ([process timeout] + (try + (let [^java.util.concurrent.Future fut + (future (.waitFor ^Process (:process process)))] + (.get fut timeout TimeUnit/MILLISECONDS)) + (catch Exception e + (if (or (instance? TimeoutException e) + (instance? TimeoutException (.getCause e))) + (do (destroy process) + :timeout) + (throw e)))))) + +(defn flush + "Flush the output stream of a process." + [process] + (let [^OutputStream in (:in process)] + (.flush in))) + +(defn done + "Close the process's output stream (sending EOF)." + [proc] + (let [^OutputStream in (:in proc)] + (.close in))) + +(defn stream-to + "Stream :out or :err from a process to an ouput stream. + Options passed are fed to clojure.java.io/copy. They are :encoding to + set the encoding and :buffer-size to set the size of the buffer. + :encoding defaults to UTF-8 and :buffer-size to 1024." + [process from to & args] + (apply io/copy (process from) to args)) + +(defn feed-from + "Feed to a process's input stream with optional. Options passed are + fed to clojure.java.io/copy. They are :encoding to set the encoding + and :buffer-size to set the size of the buffer. :encoding defaults to + UTF-8 and :buffer-size to 1024. If :flush is specified and is false, + the process will be flushed after writing." + [process from & {flush? :flush :or {flush? true} :as all}] + (apply io/copy from (:in process) all) + (when flush? (flush process))) + +(defn stream-to-string + "Streams the output of the process to a string and returns it." + [process from & args] + (with-open [writer (java.io.StringWriter.)] + (apply stream-to process from writer args) + (str writer))) + +;; The writer that Clojure wraps System/out in for *out* seems to buffer +;; things instead of writing them immediately. This wont work if you +;; really want to stream stuff, so we'll just skip it and throw our data +;; directly at System/out. +(defn stream-to-out + "Streams the output of the process to System/out" + [process from & args] + (apply stream-to process from (System/out) args)) + +(defn feed-from-string + "Feed the process some data from a string." + [process s & args] + (apply feed-from process (java.io.StringReader. s) args)) + +(defn read-line + "Read a line from a process' :out or :err." + [process from] + (binding [*in* (io/reader (from process))] + (clojure.core/read-line))) diff --git a/src/babashka/main.clj b/src/babashka/main.clj index 8317fb0c..f1c9581a 100644 --- a/src/babashka/main.clj +++ b/src/babashka/main.clj @@ -1,13 +1,19 @@ (ns babashka.main {:no-doc true} (:require - [babashka.impl.File :as File] + [babashka.impl.File :refer [file-bindings]] + [babashka.impl.System :refer [system-bindings]] + [babashka.impl.Thread :refer [thread-bindings]] + [babashka.impl.clojure.core :refer [core-bindings]] + [babashka.impl.clojure.stacktrace :refer [print-stack-trace]] + [babashka.impl.conch :refer [conch-bindings]] [babashka.impl.pipe-signal-handler :refer [handle-pipe! pipe-signal-received?]] + [babashka.impl.socket-repl :as socket-repl] + [babashka.net :as net] [clojure.edn :as edn] [clojure.java.io :as io] [clojure.java.shell :as shell] [clojure.string :as str] - [babashka.impl.socket-repl :as socket-repl] [sci.core :as sci]) (:import [sun.misc Signal] [sun.misc SignalHandler]) @@ -120,55 +126,22 @@ (str/replace x #"^#!.*" "")) (throw (Exception. (str "File does not exist: " file)))))) -(defn get-env - ([] (System/getenv)) - ([s] (System/getenv s))) - -(defn get-property - ([s] - (System/getProperty s)) - ([s d] - (System/getProperty s d))) - -(defn set-property [k v] - (System/setProperty k v)) - -(defn get-properties [] - (System/getProperties)) - -(defn exit [n] - (throw (ex-info "" {:bb/exit-code n}))) - (def bindings - (merge {'run! run! - 'shell/sh shell/sh - 'csh shell/sh ;; backwards compatibility, deprecated + (merge {'shell/sh shell/sh 'namespace namespace - 'slurp slurp - 'spit spit - 'pmap pmap - 'print print - 'pr-str pr-str - 'prn prn - 'println println - ;; clojure.java.io 'io/as-relative-path io/as-relative-path 'io/copy io/copy 'io/delete-file io/delete-file 'io/file io/file - ;; '.canRead File/canRead - ;; '.canWrite File/canWrite - ;; '.exists File/exists - ;; '.delete File/delete - + 'io/reader io/reader 'edn/read-string edn/read-string - 'System/getenv get-env - 'System/getProperty get-property - 'System/setProperty set-property - 'System/getProperties get-properties - 'System/exit exit} - File/bindings)) + 'net/wait-for-it net/wait-for-it} + core-bindings + system-bindings + file-bindings + thread-bindings + conch-bindings)) (defn read-edn [] (edn/read {;;:readers *data-readers* @@ -191,7 +164,7 @@ [& args] (handle-pipe!) #_(binding [*out* *err*] - (prn ">> args" args)) + (prn "M" (meta (get bindings 'future)))) (let [t0 (System/currentTimeMillis) {:keys [:version :shell-in :edn-in :shell-out :edn-out :help? :file :command-line-args @@ -236,25 +209,25 @@ (let [res [(do (when-not (or expression file) (throw (Exception. (str args "Babashka expected an expression. Type --help to print help.")))) (let [res (sci/eval-string expr ctx)] - (if-let [pr-f (cond shell-out println - edn-out prn)] - (if (coll? res) - (doseq [l res - :while (not (pipe-signal-received?))] - (pr-f l)) - (pr-f res)) - (prn res)))) 0]] + (if (some? res) + (if-let [pr-f (cond shell-out println + edn-out prn)] + (if (coll? res) + (doseq [l res + :while (not (pipe-signal-received?))] + (pr-f l)) + (pr-f res)) + (prn res))))) 0]] (if stream? (recur (read-next *in*)) res)))))) - (catch Exception e + (catch Throwable e (binding [*out* *err*] (let [d (ex-data e) exit-code (:bb/exit-code d)] (if exit-code [nil exit-code] - (do (when-let [msg (or (:stderr d ) - (.getMessage e))] - (println (str/trim msg))) + (do (print-stack-trace e) + (flush) [nil 1])))))))) 1) t1 (System/currentTimeMillis)] diff --git a/src/babashka/net.clj b/src/babashka/net.clj new file mode 100644 index 00000000..42e702eb --- /dev/null +++ b/src/babashka/net.clj @@ -0,0 +1,37 @@ +(ns babashka.net + (:import [java.net Socket ConnectException])) + +(set! *warn-on-reflection* true) + +(defn wait-for-it + "Waits for TCP connection to be available on host and port. Options map + supports `:timeout` and `:pause`. If `:timeout` is provided and reached, + exception will be thrown. The `:pause` option determines the time waited + between retries." + ([host port] + (wait-for-it host port nil)) + ([^String host ^long port {:keys [:timeout :pause] :as opts}] + (let [opts (merge {:host host + :port port} + opts) + t0 (System/currentTimeMillis)] + (loop [] + (let [v (try (Socket. host port) + (- (System/currentTimeMillis) t0) + (catch ConnectException _e + (let [took (- (System/currentTimeMillis) t0)] + (if (and timeout (>= took timeout)) + (throw (ex-info + (format "timeout while waiting for %s:%s" host port) + (assoc opts :took took))) + :wait-for-it.impl/try-again))))] + (if (identical? :wait-for-it.impl/try-again v) + (do (Thread/sleep (or pause 100)) + (recur)) + (assoc opts :took v))))))) + +(comment + (wait-for-it "localhost" 80) + (wait-for-it "localhost" 80 {:timeout 1000}) + (wait-for-it "google.com" 80) + ) diff --git a/test/babashka/impl/socket_repl_test.clj b/test/babashka/impl/socket_repl_test.clj index e52a0940..d41f99c1 100644 --- a/test/babashka/impl/socket_repl_test.clj +++ b/test/babashka/impl/socket_repl_test.clj @@ -43,7 +43,7 @@ "echo \"(inc 1336)\" | nc -q 1 127.0.0.1 1666"))) "1337\nbb=> "))) (testing "*in*" - (is (str/includes? (socket-command '*in*) + (is (str/includes? (socket-command "*in*") "[1 2 3]"))) (testing "*command-line-args*" (is (str/includes? (socket-command '*command-line-args*) diff --git a/test/babashka/main_test.clj b/test/babashka/main_test.clj index e01a1df5..4dbb35f3 100644 --- a/test/babashka/main_test.clj +++ b/test/babashka/main_test.clj @@ -80,7 +80,7 @@ (deftest malformed-command-line-args-test (is (thrown-with-msg? Exception #"File does not exist: non-existing\n" - (bb nil "-f" "non-existing"))) + (bb nil "-f" "non-existing"))) (is (thrown-with-msg? Exception #"expression" (bb nil)))) @@ -136,3 +136,19 @@ (let [out (:out (sh "bash" "-c" "yes | ./bb -i '(take 2 *in*)'")) out (edn/read-string out)] (is (= '("y" "y") out))))) + +(deftest future-test + (is (= 6 (bb nil "@(future (+ 1 2 3))")))) + +(deftest conch-test + (is (str/includes? (bb nil "(->> (conch/proc \"ls\") (conch/stream-to-string :out))") + "LICENSE"))) + +(deftest wait-for-it-test + (is (thrown-with-msg? + Exception + #"timeout" + (bb nil "(def web-server (conch/proc \"python\" \"-m\" \"SimpleHTTPServer\" \"7171\")) + (net/wait-for-it \"127.0.0.1\" 7171) + (conch/destroy web-server) + (net/wait-for-it \"localhost\" 7172 {:timeout 50})"))))