From f2cfdff8998ca8bae116ff9d8a60f2b7a2d0da3e Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Wed, 23 Mar 2022 05:26:58 -0600 Subject: [PATCH] Feature: declarative pods (#44) * Use non-deprecated string->int method * Remove unused next-pod-id * Support declarative pods loaded on require * Wait for pod shutdown in load-pod-metadata * Type hint a File return value to avoid reflection * Return pod metadata instead of putting in ctx * Fix local pod loading & support :cache opt * Document :pods in bb.edn * Cache local pods metadata in project .babashka dir * Pass pod resolve-fn to describe->metadata Not only was this just a bug, but the accidental reference to clojure.core/resolve ballooned the final bb image size to >110MB! --- README.md | 27 ++++++ src/babashka/pods/impl.clj | 144 +++++++++++++++++----------- src/babashka/pods/impl/resolver.clj | 1 + src/babashka/pods/sci.clj | 47 ++++++++- 4 files changed, 162 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 3bdaf74..e67f6d0 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,33 @@ On the JVM: When calling `load-pod` with a string or vector of strings, the pod is looked up on the local file system (either using the PATH, or using an absolute path). When it is called with a qualified symbol and a version - like `(load-pod 'org.babashka/aws "0.0.5")` then it will be looked up in and downloaded from the [pod-registry](https://github.com/babashka/pod-registry). +### In a babashka project + +As of babashka 0.8.0 you can declare the pods your babashka project uses in your `bb.edn` file like so: + +```clojure +:pods {org.babashka/hsqldb {:version "0.1.0"} ; will be downloaded from the babashka pod registry + my.local/pod {:path "../pod-my-local/my-pod-binary" + :cache false}} ; optionally disable namespace caching if you're actively working on this pod +``` + +Then you can just require the pods in your code like any other clojure lib: + +```clojure +(ns my.project + (:require [pod.babashka.hsqldb :as sql] + [my.local.pod :as my-pod])) + +(def db "jdbc:hsqldb:mem:testdb;sql.syntax_mys=true") +(sql/execute! db ["create table foo ( foo int );"]) +;;=> [#:next.jdbc{:update-count 0}] + +(my-pod/do-a-thing "foo") +;;=> "something" +``` + +The pods will then be loaded on demand when you require them. No need to call `load-pod` explicitly. + ## Sci To use pods in a [sci](https://github.com/borkdude/sci) based project, see diff --git a/src/babashka/pods/impl.clj b/src/babashka/pods/impl.clj index 1d5ad40..4c4312b 100644 --- a/src/babashka/pods/impl.clj +++ b/src/babashka/pods/impl.clj @@ -248,27 +248,24 @@ (defn lookup-pod [pod-id] (get @pods pod-id)) +(defn destroy* [{:keys [:stdin :process :ops]}] + (if (contains? ops :shutdown) + (do (write stdin + {"op" "shutdown" + "id" (next-id)}) + (.waitFor ^Process process)) + (.destroy ^Process process))) + (defn destroy [pod-id-or-pod] (let [pod-id (get-pod-id pod-id-or-pod)] (when-let [pod (lookup-pod pod-id)] - (if (contains? (:ops pod) :shutdown) - (do (write (:stdin pod) - {"op" "shutdown" - "id" (next-id)}) - (.waitFor ^Process (:process pod))) - (.destroy ^Process (:process pod))) + (destroy* pod) (when-let [rns (:remove-ns pod)] (doseq [[ns-name _] (:namespaces pod)] (rns ns-name)))) (swap! pods dissoc pod-id) nil)) -(def next-pod-id - (let [counter (atom 0)] - (fn [] - (let [[o _] (swap-vals! counter inc)] - o)))) - (def bytes->symbol (comp symbol bytes->string)) @@ -310,59 +307,94 @@ (let [s (slurp f)] (when (str/ends-with? s "\n") (str/trim s))))] - (Integer. s) + (Integer/parseInt s) (recur))))) (defn debug [& strs] (binding [*out* *err*] (println (str/join " " (map pr-str strs))))) +(defn resolve-pod [pod-spec {:keys [:version :path :force] :as opts}] + (let [resolved (when (and (qualified-symbol? pod-spec) version) + (resolver/resolve pod-spec version force)) + opts (if resolved + (if-let [extra-opts (:options resolved)] + (merge opts extra-opts) + opts) + opts) + pod-spec (cond + resolved [(:executable resolved)] + path [path] + (string? pod-spec) [pod-spec] + :else pod-spec)] + {:pod-spec pod-spec, :opts opts})) + +(defn run-pod [pod-spec {:keys [:transport] :as _opts}] + (let [pb (ProcessBuilder. ^java.util.List pod-spec) + socket? (identical? :socket transport) + _ (if socket? + (.inheritIO pb) + (.redirectError pb java.lang.ProcessBuilder$Redirect/INHERIT)) + _ (cond-> (doto (.environment pb) + (.put "BABASHKA_POD" "true")) + socket? (.put "BABASHKA_POD_TRANSPORT" "socket")) + p (.start pb) + port-file (when socket? (port-file (.pid p))) + socket-port (when socket? (read-port port-file)) + [socket stdin stdout] + (if socket? + (let [^Socket socket + (loop [] + (if-let [sock (try (create-socket "localhost" socket-port) + (catch java.net.ConnectException _ + nil))] + sock + (recur)))] + [socket + (.getOutputStream socket) + (PushbackInputStream. (.getInputStream socket))]) + [nil (.getOutputStream p) (java.io.PushbackInputStream. (.getInputStream p))])] + {:process p + :socket socket + :stdin stdin + :stdout stdout})) + +(defn describe-pod [{:keys [:stdin :stdout]}] + (write stdin {"op" "describe" + "id" (next-id)}) + (read stdout)) + +(defn describe->ops [describe-reply] + (some->> (get describe-reply "ops") keys (map keyword) set)) + +(defn describe->metadata [describe-reply resolve-fn] + (let [format (-> (get describe-reply "format") bytes->string keyword) + ops (describe->ops describe-reply) + readers (when (identical? :edn format) + (read-readers describe-reply resolve-fn))] + {:format format, :ops ops, :readers readers})) + +(defn load-pod-metadata [pod-spec opts] + (let [{:keys [:pod-spec :opts]} (resolve-pod pod-spec opts) + running-pod (run-pod pod-spec opts) + describe-reply (describe-pod running-pod) + ops (describe->ops describe-reply)] + (destroy* (assoc running-pod :ops ops)) + describe-reply)) + (defn load-pod ([pod-spec] (load-pod pod-spec nil)) ([pod-spec opts] - (let [{:keys [:version :force]} opts - resolved (when (qualified-symbol? pod-spec) - (resolver/resolve pod-spec version force)) - opts (if resolved - (if-let [extra-opts (:options resolved)] - (merge opts extra-opts) - opts) - opts) - {:keys [:remove-ns :resolve :transport]} opts - pod-spec (cond resolved [(:executable resolved)] - (string? pod-spec) [pod-spec] - :else pod-spec) - pb (ProcessBuilder. ^java.util.List pod-spec) - socket? (identical? :socket transport) - _ (if socket? - (.inheritIO pb) - (.redirectError pb java.lang.ProcessBuilder$Redirect/INHERIT)) - _ (cond-> (doto (.environment pb) - (.put "BABASHKA_POD" "true")) - socket? (.put "BABASHKA_POD_TRANSPORT" "socket")) - p (.start pb) - port-file (when socket? (port-file (.pid p))) - socket-port (when socket? (read-port port-file)) - [socket stdin stdout] - (if socket? - (let [^Socket socket - (loop [] - (if-let [sock (try (create-socket "localhost" socket-port) - (catch java.net.ConnectException _ - nil))] - sock - (recur)))] - [socket - (.getOutputStream socket) - (PushbackInputStream. (.getInputStream socket))]) - [nil (.getOutputStream p) (java.io.PushbackInputStream. (.getInputStream p))]) - _ (write stdin {"op" "describe" - "id" (next-id)}) - reply (read stdout) - format (-> (get reply "format") bytes->string keyword) - ops (some->> (get reply "ops") keys (map keyword) set) - readers (when (identical? :edn format) - (read-readers reply resolve)) + (let [{:keys [:pod-spec :opts]} (resolve-pod pod-spec opts) + {:keys [:remove-ns :resolve]} opts + + {p :process, stdin :stdin, stdout :stdout, socket :socket + :as running-pod} + (run-pod pod-spec opts) + + reply (or (:metadata opts) + (describe-pod running-pod)) + {:keys [:format :ops :readers]} (describe->metadata reply resolve) pod {:process p :pod-spec pod-spec :stdin stdin diff --git a/src/babashka/pods/impl/resolver.clj b/src/babashka/pods/impl/resolver.clj index 55d6000..ffec661 100644 --- a/src/babashka/pods/impl/resolver.clj +++ b/src/babashka/pods/impl/resolver.clj @@ -18,6 +18,7 @@ (def os {:os/name (System/getProperty "os.name") :os/arch (let [arch (System/getProperty "os.arch")] (normalize-arch arch))}) + (defn warn [& strs] (binding [*out* *err*] (apply println strs))) diff --git a/src/babashka/pods/sci.clj b/src/babashka/pods/sci.clj index 437aea5..ad0fea8 100644 --- a/src/babashka/pods/sci.clj +++ b/src/babashka/pods/sci.clj @@ -1,6 +1,12 @@ (ns babashka.pods.sci (:require [babashka.pods.impl :as impl] - [sci.core :as sci])) + [sci.core :as sci] + [clojure.java.io :as io] + [babashka.pods.impl.resolver :as resolver] + [babashka.impl.common :as common]) + (:import (java.io PushbackInputStream File))) + +(set! *warn-on-reflection* true) (defn- process-namespace [ctx {:keys [:name :vars]}] (let [env (:env ctx) @@ -19,6 +25,45 @@ (string? var-value) (sci/eval-string* ctx var-value)))))) +(defn metadata-cache-file ^File [pod-spec {:keys [:version :path]}] + (if version + (io/file (resolver/cache-dir {:pod/name pod-spec :pod/version version}) + "metadata.cache") + (let [bb-edn-file (-> @common/bb-edn :file io/file) + config-dir (.getParentFile bb-edn-file) + cache-dir (io/file config-dir ".babashka") + pod-file (-> path io/file .getName) + cache-file (io/file cache-dir (str pod-file ".metadata.cache"))] + cache-file))) + +(defn load-metadata-from-cache [pod-spec opts] + (let [cache-file (metadata-cache-file pod-spec opts)] + (when (.exists cache-file) + (with-open [r (PushbackInputStream. (io/input-stream cache-file))] + (impl/read r))))) + +(defn load-pod-metadata* [pod-spec {:keys [:version :cache] :as opts}] + (let [metadata (impl/load-pod-metadata pod-spec opts) + cache-file (when cache (metadata-cache-file pod-spec opts))] + (when cache-file + (io/make-parents cache-file) + (with-open [w (io/output-stream cache-file)] + (impl/write w metadata))) + metadata)) + +(defn load-pod-metadata [pod-spec {:keys [:cache] :as opts}] + (let [metadata + (if-let [cached-metadata (when cache + (load-metadata-from-cache pod-spec opts))] + cached-metadata + (load-pod-metadata* pod-spec opts))] + (reduce + (fn [pod-namespaces ns] + (let [ns-sym (-> ns (get "name") impl/bytes->string symbol)] + (assoc pod-namespaces ns-sym {:pod-spec pod-spec + :opts (assoc opts :metadata metadata)}))) + {} (get metadata "namespaces")))) + (defn load-pod ([ctx pod-spec] (load-pod ctx pod-spec nil)) ([ctx pod-spec version opts] (load-pod ctx pod-spec (assoc opts :version version)))