luciano/pods
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feature: declarative pods loaded at require time (#43)

Browse source 
* Use non-deprecated string->int method

* Remove unused next-pod-id

* Support declarative pods loaded on require

* Wait for pod shutdown in load-pod-metadata

* Type hint a File return value to avoid reflection

* Return pod metadata instead of putting in ctx

* Fix local pod loading & support :cache opt

* Document :pods in bb.edn

* Cache local pods metadata in project .babashka dir
This commit is contained in:
Wes Morgan 2022-03-21 13:46:57 -06:00 committed by GitHub
parent 538fc6f414
commit 8f059da7e6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 162 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
README.md
View file

@ -76,6 +76,33 @@ On the JVM:
When calling `load-pod` with a string or vector of strings, the pod is looked up on the local file system (either using the PATH, or using an absolute path). When it is called with a qualified symbol and a version - like `(load-pod 'org.babashka/aws "0.0.5")` then it will be looked up in and downloaded from the [pod-registry](https://github.com/babashka/pod-registry).
### In a babashka project
As of babashka 0.8.0 you can declare the pods your babashka project uses in your `bb.edn` file like so:
```clojure
:pods {org.babashka/hsqldb {:version "0.1.0"} ; will be downloaded from the babashka pod registry
my.local/pod {:path "../pod-my-local/my-pod-binary"
:cache false}} ; optionally disable namespace caching if you're actively working on this pod
```
Then you can just require the pods in your code like any other clojure lib:
```clojure
(ns my.project
(:require [pod.babashka.hsqldb :as sql]
[my.local.pod :as my-pod]))
(def db "jdbc:hsqldb:mem:testdb;sql.syntax_mys=true")
(sql/execute! db ["create table foo ( foo int );"])
;;=> [#:next.jdbc{:update-count 0}]
(my-pod/do-a-thing "foo")
;;=> "something"
```
The pods will then be loaded on demand when you require them. No need to call `load-pod` explicitly.
## Sci
To use pods in a [sci](https://github.com/borkdude/sci) based project, see

144
src/babashka/pods/impl.clj
View file

@ -248,27 +248,24 @@
(defn lookup-pod [pod-id]
(get @pods pod-id))
(defn destroy* [{:keys [:stdin :process :ops]}]
(if (contains? ops :shutdown)
(do (write stdin
{"op" "shutdown"
"id" (next-id)})
(.waitFor ^Process process))
(.destroy ^Process process)))
(defn destroy [pod-id-or-pod]
(let [pod-id (get-pod-id pod-id-or-pod)]
(when-let [pod (lookup-pod pod-id)]
(if (contains? (:ops pod) :shutdown)
(do (write (:stdin pod)
{"op" "shutdown"
"id" (next-id)})
(.waitFor ^Process (:process pod)))
(.destroy ^Process (:process pod)))
(destroy* pod)
(when-let [rns (:remove-ns pod)]
(doseq [[ns-name _] (:namespaces pod)]
(rns ns-name))))
(swap! pods dissoc pod-id)
nil))
(def next-pod-id
(let [counter (atom 0)]
(fn []
(let [[o _] (swap-vals! counter inc)]
o))))
(def bytes->symbol
(comp symbol bytes->string))
@ -310,59 +307,94 @@
(let [s (slurp f)]
(when (str/ends-with? s "\n")
(str/trim s))))]
(Integer. s)
(Integer/parseInt s)
(recur)))))
(defn debug [& strs]
(binding [*out* *err*]
(println (str/join " " (map pr-str strs)))))
(defn resolve-pod [pod-spec {:keys [:version :path :force] :as opts}]
(let [resolved (when (and (qualified-symbol? pod-spec) version)
(resolver/resolve pod-spec version force))
opts (if resolved
(if-let [extra-opts (:options resolved)]
(merge opts extra-opts)
opts)
opts)
pod-spec (cond
resolved [(:executable resolved)]
path [path]
(string? pod-spec) [pod-spec]
:else pod-spec)]
{:pod-spec pod-spec, :opts opts}))
(defn run-pod [pod-spec {:keys [:transport] :as _opts}]
(let [pb (ProcessBuilder. ^java.util.List pod-spec)
socket? (identical? :socket transport)
_ (if socket?
(.inheritIO pb)
(.redirectError pb java.lang.ProcessBuilder$Redirect/INHERIT))
_ (cond-> (doto (.environment pb)
(.put "BABASHKA_POD" "true"))
socket? (.put "BABASHKA_POD_TRANSPORT" "socket"))
p (.start pb)
port-file (when socket? (port-file (.pid p)))
socket-port (when socket? (read-port port-file))
[socket stdin stdout]
(if socket?
(let [^Socket socket
(loop []
(if-let [sock (try (create-socket "localhost" socket-port)
(catch java.net.ConnectException _
nil))]
sock
(recur)))]
[socket
(.getOutputStream socket)
(PushbackInputStream. (.getInputStream socket))])
[nil (.getOutputStream p) (java.io.PushbackInputStream. (.getInputStream p))])]
{:process p
:socket socket
:stdin stdin
:stdout stdout}))
(defn describe-pod [{:keys [:stdin :stdout]}]
(write stdin {"op" "describe"
"id" (next-id)})
(read stdout))
(defn describe->ops [describe-reply]
(some->> (get describe-reply "ops") keys (map keyword) set))
(defn describe->metadata [describe-reply]
(let [format (-> (get describe-reply "format") bytes->string keyword)
ops (describe->ops describe-reply)
readers (when (identical? :edn format)
(read-readers describe-reply resolve))]
{:format format, :ops ops, :readers readers}))
(defn load-pod-metadata [pod-spec opts]
(let [{:keys [:pod-spec :opts]} (resolve-pod pod-spec opts)
running-pod (run-pod pod-spec opts)
describe-reply (describe-pod running-pod)
ops (describe->ops describe-reply)]
(destroy* (assoc running-pod :ops ops))
describe-reply))
(defn load-pod
([pod-spec] (load-pod pod-spec nil))
([pod-spec opts]
(let [{:keys [:version :force]} opts
resolved (when (qualified-symbol? pod-spec)
(resolver/resolve pod-spec version force))
opts (if resolved
(if-let [extra-opts (:options resolved)]
(merge opts extra-opts)
opts)
opts)
{:keys [:remove-ns :resolve :transport]} opts
pod-spec (cond resolved [(:executable resolved)]
(string? pod-spec) [pod-spec]
:else pod-spec)
pb (ProcessBuilder. ^java.util.List pod-spec)
socket? (identical? :socket transport)
_ (if socket?
(.inheritIO pb)
(.redirectError pb java.lang.ProcessBuilder$Redirect/INHERIT))
_ (cond-> (doto (.environment pb)
(.put "BABASHKA_POD" "true"))
socket? (.put "BABASHKA_POD_TRANSPORT" "socket"))
p (.start pb)
port-file (when socket? (port-file (.pid p)))
socket-port (when socket? (read-port port-file))
[socket stdin stdout]
(if socket?
(let [^Socket socket
(loop []
(if-let [sock (try (create-socket "localhost" socket-port)
(catch java.net.ConnectException _
nil))]
sock
(recur)))]
[socket
(.getOutputStream socket)
(PushbackInputStream. (.getInputStream socket))])
[nil (.getOutputStream p) (java.io.PushbackInputStream. (.getInputStream p))])
_ (write stdin {"op" "describe"
"id" (next-id)})
reply (read stdout)
format (-> (get reply "format") bytes->string keyword)
ops (some->> (get reply "ops") keys (map keyword) set)
readers (when (identical? :edn format)
(read-readers reply resolve))
(let [{:keys [:pod-spec :opts]} (resolve-pod pod-spec opts)
{:keys [:remove-ns :resolve]} opts
{p :process, stdin :stdin, stdout :stdout, socket :socket
:as running-pod}
(run-pod pod-spec opts)
reply (or (:metadata opts)
(describe-pod running-pod))
{:keys [:format :ops :readers]} (describe->metadata reply)
pod {:process p
:pod-spec pod-spec
:stdin stdin

1
src/babashka/pods/impl/resolver.clj
View file

@ -18,6 +18,7 @@
(def os {:os/name (System/getProperty "os.name")
:os/arch (let [arch (System/getProperty "os.arch")]
(normalize-arch arch))})
(defn warn [& strs]
(binding [*out* *err*]
(apply println strs)))

47
src/babashka/pods/sci.clj
View file

@ -1,6 +1,12 @@
(ns babashka.pods.sci
(:require [babashka.pods.impl :as impl]
[sci.core :as sci]))
[sci.core :as sci]
[clojure.java.io :as io]
[babashka.pods.impl.resolver :as resolver]
[babashka.impl.common :as common])
(:import (java.io PushbackInputStream File)))
(set! *warn-on-reflection* true)
(defn- process-namespace [ctx {:keys [:name :vars]}]
(let [env (:env ctx)
@ -19,6 +25,45 @@
(string? var-value)
(sci/eval-string* ctx var-value))))))
(defn metadata-cache-file ^File [pod-spec {:keys [:version :path]}]
(if version
(io/file (resolver/cache-dir {:pod/name pod-spec :pod/version version})
"metadata.cache")
(let [bb-edn-file (-> @common/bb-edn :file io/file)
config-dir (.getParentFile bb-edn-file)
cache-dir (io/file config-dir ".babashka")
pod-file (-> path io/file .getName)
cache-file (io/file cache-dir (str pod-file ".metadata.cache"))]
cache-file)))
(defn load-metadata-from-cache [pod-spec opts]
(let [cache-file (metadata-cache-file pod-spec opts)]
(when (.exists cache-file)
(with-open [r (PushbackInputStream. (io/input-stream cache-file))]
(impl/read r)))))
(defn load-pod-metadata* [pod-spec {:keys [:version :cache] :as opts}]
(let [metadata (impl/load-pod-metadata pod-spec opts)
cache-file (when cache (metadata-cache-file pod-spec opts))]
(when cache-file
(io/make-parents cache-file)
(with-open [w (io/output-stream cache-file)]
(impl/write w metadata)))
metadata))
(defn load-pod-metadata [pod-spec {:keys [:cache] :as opts}]
(let [metadata
(if-let [cached-metadata (when cache
(load-metadata-from-cache pod-spec opts))]
cached-metadata
(load-pod-metadata* pod-spec opts))]
(reduce
(fn [pod-namespaces ns]
(let [ns-sym (-> ns (get "name") impl/bytes->string symbol)]
(assoc pod-namespaces ns-sym {:pod-spec pod-spec
:opts (assoc opts :metadata metadata)})))
{} (get metadata "namespaces"))))
(defn load-pod
([ctx pod-spec] (load-pod ctx pod-spec nil))
([ctx pod-spec version opts] (load-pod ctx pod-spec (assoc opts :version version)))