Feature: declarative pods (#44)

* Use non-deprecated string->int method

* Remove unused next-pod-id

* Support declarative pods loaded on require

* Wait for pod shutdown in load-pod-metadata

* Type hint a File return value to avoid reflection

* Return pod metadata instead of putting in ctx

* Fix local pod loading & support :cache opt

* Document :pods in bb.edn

* Cache local pods metadata in project .babashka dir

* Pass pod resolve-fn to describe->metadata

Not only was this just a bug, but the accidental reference to clojure.core/resolve ballooned the final bb image size to >110MB!

README.md

@@ -76,6 +76,33 @@ On the JVM:
 
 When calling `load-pod` with a string or vector of strings, the pod is looked up on the local file system (either using the PATH, or using an absolute path). When it is called with a qualified symbol and a version - like `(load-pod 'org.babashka/aws "0.0.5")` then it will be looked up in and downloaded from the [pod-registry](https://github.com/babashka/pod-registry).
 
+### In a babashka project
+
+As of babashka 0.8.0 you can declare the pods your babashka project uses in your `bb.edn` file like so:
+
+```clojure
+:pods {org.babashka/hsqldb {:version "0.1.0"} ; will be downloaded from the babashka pod registry
+       my.local/pod {:path "../pod-my-local/my-pod-binary"
+                     :cache false}} ; optionally disable namespace caching if you're actively working on this pod
+```
+
+Then you can just require the pods in your code like any other clojure lib:
+
+```clojure
+(ns my.project
+  (:require [pod.babashka.hsqldb :as sql]
+            [my.local.pod :as my-pod]))
+
+(def db "jdbc:hsqldb:mem:testdb;sql.syntax_mys=true")
+(sql/execute! db ["create table foo ( foo int );"])
+;;=> [#:next.jdbc{:update-count 0}]
+
+(my-pod/do-a-thing "foo")
+;;=> "something"
+```
+
+The pods will then be loaded on demand when you require them. No need to call `load-pod` explicitly.
+
 ## Sci
 
 To use pods in a [sci](https://github.com/borkdude/sci) based project, see

src/babashka/pods/impl.clj

@@ -248,27 +248,24 @@
 (defn lookup-pod [pod-id]
   (get @pods pod-id))
 
+(defn destroy* [{:keys [:stdin :process :ops]}]
+  (if (contains? ops :shutdown)
+    (do (write stdin
+               {"op" "shutdown"
+                "id" (next-id)})
+        (.waitFor ^Process process))
+    (.destroy ^Process process)))
+
 (defn destroy [pod-id-or-pod]
   (let [pod-id (get-pod-id pod-id-or-pod)]
     (when-let [pod (lookup-pod pod-id)]
-      (if (contains? (:ops pod) :shutdown)
+      (destroy* pod)
-        (do (write (:stdin pod)
-                   {"op" "shutdown"
-                    "id" (next-id)})
-            (.waitFor ^Process (:process pod)))
-        (.destroy ^Process (:process pod)))
       (when-let [rns (:remove-ns pod)]
         (doseq [[ns-name _] (:namespaces pod)]
           (rns ns-name))))
     (swap! pods dissoc pod-id)
     nil))
 
-(def next-pod-id
-  (let [counter (atom 0)]
-    (fn []
-      (let [[o _] (swap-vals! counter inc)]
-        o))))
-
 (def bytes->symbol
   (comp symbol bytes->string))
 
@@ -310,59 +307,94 @@
                    (let [s (slurp f)]
                      (when (str/ends-with? s "\n")
                        (str/trim s))))]
-        (Integer. s)
+        (Integer/parseInt s)
         (recur)))))
 
 (defn debug [& strs]
   (binding [*out* *err*]
     (println (str/join " " (map pr-str strs)))))
 
+(defn resolve-pod [pod-spec {:keys [:version :path :force] :as opts}]
+  (let [resolved (when (and (qualified-symbol? pod-spec) version)
+                   (resolver/resolve pod-spec version force))
+        opts (if resolved
+               (if-let [extra-opts (:options resolved)]
+                 (merge opts extra-opts)
+                 opts)
+               opts)
+        pod-spec (cond
+                   resolved [(:executable resolved)]
+                   path [path]
+                   (string? pod-spec) [pod-spec]
+                   :else pod-spec)]
+    {:pod-spec pod-spec, :opts opts}))
+
+(defn run-pod [pod-spec {:keys [:transport] :as _opts}]
+  (let [pb (ProcessBuilder. ^java.util.List pod-spec)
+        socket? (identical? :socket transport)
+        _ (if socket?
+            (.inheritIO pb)
+            (.redirectError pb java.lang.ProcessBuilder$Redirect/INHERIT))
+        _ (cond-> (doto (.environment pb)
+                    (.put "BABASHKA_POD" "true"))
+                 socket? (.put "BABASHKA_POD_TRANSPORT" "socket"))
+        p (.start pb)
+        port-file (when socket? (port-file (.pid p)))
+        socket-port (when socket? (read-port port-file))
+        [socket stdin stdout]
+        (if socket?
+          (let [^Socket socket
+                (loop []
+                  (if-let [sock (try (create-socket "localhost" socket-port)
+                                     (catch java.net.ConnectException _
+                                       nil))]
+                    sock
+                    (recur)))]
+            [socket
+             (.getOutputStream socket)
+             (PushbackInputStream. (.getInputStream socket))])
+          [nil (.getOutputStream p) (java.io.PushbackInputStream. (.getInputStream p))])]
+    {:process p
+     :socket socket
+     :stdin stdin
+     :stdout stdout}))
+
+(defn describe-pod [{:keys [:stdin :stdout]}]
+  (write stdin {"op" "describe"
+                "id" (next-id)})
+  (read stdout))
+
+(defn describe->ops [describe-reply]
+  (some->> (get describe-reply "ops") keys (map keyword) set))
+
+(defn describe->metadata [describe-reply resolve-fn]
+  (let [format (-> (get describe-reply "format") bytes->string keyword)
+        ops (describe->ops describe-reply)
+        readers (when (identical? :edn format)
+                  (read-readers describe-reply resolve-fn))]
+    {:format format, :ops ops, :readers readers}))
+
+(defn load-pod-metadata [pod-spec opts]
+  (let [{:keys [:pod-spec :opts]} (resolve-pod pod-spec opts)
+        running-pod (run-pod pod-spec opts)
+        describe-reply (describe-pod running-pod)
+        ops (describe->ops describe-reply)]
+    (destroy* (assoc running-pod :ops ops))
+    describe-reply))
+
 (defn load-pod
   ([pod-spec] (load-pod pod-spec nil))
   ([pod-spec opts]
-   (let [{:keys [:version :force]} opts
+   (let [{:keys [:pod-spec :opts]} (resolve-pod pod-spec opts)
-         resolved (when (qualified-symbol? pod-spec)
+         {:keys [:remove-ns :resolve]} opts
-                    (resolver/resolve pod-spec version force))
+
-         opts (if resolved
+         {p :process, stdin :stdin, stdout :stdout, socket :socket
-                (if-let [extra-opts (:options resolved)]
+          :as running-pod}
-                  (merge opts extra-opts)
+         (run-pod pod-spec opts)
-                  opts)
+
-                opts)
+         reply (or (:metadata opts)
-         {:keys [:remove-ns :resolve :transport]} opts
+                   (describe-pod running-pod))
-         pod-spec (cond resolved [(:executable resolved)]
+         {:keys [:format :ops :readers]} (describe->metadata reply resolve)
-                        (string? pod-spec) [pod-spec]
-                        :else pod-spec)
-         pb (ProcessBuilder. ^java.util.List pod-spec)
-         socket? (identical? :socket transport)
-         _ (if socket?
-             (.inheritIO pb)
-             (.redirectError pb java.lang.ProcessBuilder$Redirect/INHERIT))
-         _ (cond-> (doto (.environment pb)
-                     (.put "BABASHKA_POD" "true"))
-             socket? (.put "BABASHKA_POD_TRANSPORT" "socket"))
-         p (.start pb)
-         port-file (when socket? (port-file (.pid p)))
-         socket-port (when socket? (read-port port-file))
-         [socket stdin stdout]
-         (if socket?
-           (let [^Socket socket
-                 (loop []
-                   (if-let [sock (try (create-socket "localhost" socket-port)
-                                      (catch java.net.ConnectException _
-                                        nil))]
-                     sock
-                     (recur)))]
-             [socket
-              (.getOutputStream socket)
-              (PushbackInputStream. (.getInputStream socket))])
-           [nil (.getOutputStream p) (java.io.PushbackInputStream. (.getInputStream p))])
-         _ (write stdin {"op" "describe"
-                         "id" (next-id)})
-         reply (read stdout)
-         format (-> (get reply "format") bytes->string keyword)
-         ops (some->> (get reply "ops") keys (map keyword) set)
-         readers (when (identical? :edn format)
-                   (read-readers reply resolve))
          pod {:process p
               :pod-spec pod-spec
               :stdin stdin

src/babashka/pods/impl/resolver.clj

@@ -18,6 +18,7 @@
 (def os {:os/name (System/getProperty "os.name")
          :os/arch (let [arch (System/getProperty "os.arch")]
                     (normalize-arch arch))})
+
 (defn warn [& strs]
   (binding [*out* *err*]
     (apply println strs)))

src/babashka/pods/sci.clj

@@ -1,6 +1,12 @@
 (ns babashka.pods.sci
   (:require [babashka.pods.impl :as impl]
-            [sci.core :as sci]))
+            [sci.core :as sci]
+            [clojure.java.io :as io]
+            [babashka.pods.impl.resolver :as resolver]
+            [babashka.impl.common :as common])
+  (:import (java.io PushbackInputStream File)))
+
+(set! *warn-on-reflection* true)
 
 (defn- process-namespace [ctx {:keys [:name :vars]}]
   (let [env (:env ctx)
@@ -19,6 +25,45 @@
               (string? var-value)
               (sci/eval-string* ctx var-value))))))
 
+(defn metadata-cache-file ^File [pod-spec {:keys [:version :path]}]
+  (if version
+    (io/file (resolver/cache-dir {:pod/name pod-spec :pod/version version})
+             "metadata.cache")
+    (let [bb-edn-file (-> @common/bb-edn :file io/file)
+          config-dir (.getParentFile bb-edn-file)
+          cache-dir (io/file config-dir ".babashka")
+          pod-file (-> path io/file .getName)
+          cache-file (io/file cache-dir (str pod-file ".metadata.cache"))]
+      cache-file)))
+
+(defn load-metadata-from-cache [pod-spec opts]
+  (let [cache-file (metadata-cache-file pod-spec opts)]
+    (when (.exists cache-file)
+      (with-open [r (PushbackInputStream. (io/input-stream cache-file))]
+        (impl/read r)))))
+
+(defn load-pod-metadata* [pod-spec {:keys [:version :cache] :as opts}]
+  (let [metadata (impl/load-pod-metadata pod-spec opts)
+        cache-file (when cache (metadata-cache-file pod-spec opts))]
+    (when cache-file
+      (io/make-parents cache-file)
+      (with-open [w (io/output-stream cache-file)]
+        (impl/write w metadata)))
+    metadata))
+
+(defn load-pod-metadata [pod-spec {:keys [:cache] :as opts}]
+  (let [metadata
+        (if-let [cached-metadata (when cache
+                                   (load-metadata-from-cache pod-spec opts))]
+          cached-metadata
+          (load-pod-metadata* pod-spec opts))]
+    (reduce
+      (fn [pod-namespaces ns]
+        (let [ns-sym (-> ns (get "name") impl/bytes->string symbol)]
+          (assoc pod-namespaces ns-sym {:pod-spec pod-spec
+                                        :opts (assoc opts :metadata metadata)})))
+      {} (get metadata "namespaces"))))
+
 (defn load-pod
   ([ctx pod-spec] (load-pod ctx pod-spec nil))
   ([ctx pod-spec version opts] (load-pod ctx pod-spec (assoc opts :version version)))