Merge branch 'master' into feature/local-manifests

...as a part of supporting library-bootstrapped pods

.gitignore

@@ -3,7 +3,3 @@
 .lein-failures
 /pom.xml
 .lein-repl-history
-.cache
-.clj-kondo/babashka
-.clj-kondo/rewrite-clj
-src/scratch.clj

CHANGELOG.md

@@ -2,20 +2,6 @@
 
 ## Unreleased
 
-- [#63](https://github.com/babashka/pods/issues/63): create directory before un-tarring
+- #49: don't log socket closed exception
-- [#59](https://github.com/babashka/pods/issues/59): delete port file on exit
-- [#65](https://github.com/babashka/pods/issues/65): fix warnings when defining var with core name in JVM
-- [#66](https://github.com/babashka/pods/issues/66): Allow metadata on fn arguments for transit+json
 
-## v0.2.0
+## ...
-
-- [#61](https://github.com/babashka/pods/issues/61): add transit as explicit JVM dependency
-- [#60](https://github.com/babashka/pods/issues/60): transform pod reader error into exception of caller
-- Switch "out" and "err" messages to print and flush instead of `println` ([@justone](https://github.com/justone))
-- Set TCP_NODELAY on transport socket ([@retrogradeorbit](https://github.com/retrogradeorbit))
--  Allow env vars OS_NAME & OS_ARCH to override os props ([@cap10morgan](https://github.com/cap10morgan))
-- [#49](https://github.com/babashka/pods/issues/49): don't log socket closed exception
-
-## v0.1.0
-
-Initial version

README.md

@@ -77,7 +77,7 @@ On the JVM:
 When calling `load-pod` with a string or vector of strings (or declaring it in your `bb.edn`),
 the pod is looked up on the local file system (either using the PATH, or using an absolute path).
 When it is called with a qualified symbol and a version - like `(load-pod 'org.babashka/aws "0.0.5")`
-then it will be looked up in and downloaded from the [pod-registry](https://github.com/babashka/pod-registry). You can customize the file system location that `load-pod` will use by setting the `BABASHKA_PODS_DIR` environment variable.
+then it will be looked up in and downloaded from the [pod-registry](https://github.com/babashka/pod-registry).
 
 By default babashka will search for a pod binary matching your system's OS and arch. If you want to download
 pods for a different OS / arch (e.g. for deployment to servers), you can set one or both of the following
@@ -130,7 +130,7 @@ light weight replacement for native interop (JNI, JNA, etc.).
 
 ### Examples
 
-Beyond the already available pods mentioned above, educational examples of pods
+Beyond the already available pods mentioned above, eductional examples of pods
 can be found [here](examples):
 
 - [pod-lispyclouds-sqlite](examples/pod-lispyclouds-sqlite): a pod that
@@ -228,7 +228,7 @@ JSON. It also declares that the pod exposes one namespace,
 
 To encode payloads in EDN use `"edn"` and for Transit JSON use `"transit+json"`.
 
-The pod encodes the above map to bencode and writes it to stdout. The pod client
+The pod encodes the above map to bencode and writes it to stdoud. The pod client
 reads this message from the pod's stdout.
 
 Upon receiving this message, the pod client creates these namespaces and vars.
@@ -376,15 +376,11 @@ nil
 
 #### Metadata
 
-**From pod to pod client**
-
-*Fixed Metadata on vars*
-
 Pods may attach metadata to functions and macros by sending data to the pod client
 in a `"meta"` field as part of a `"var"` section. The metadata must be an appropriate
 map, encoded as an EDN string. This is only applicable to vars in the pod and will be
 ignored if the var refers to Client-side code, since metadata can already be defined
-in those code blocks (see 'Dynamic Metadata' below to enable the encoding of metadata).
+in those code blocks.
 
 For example, a pod can define a function called `add`:
 
@@ -396,33 +392,6 @@ For example, a pod can define a function called `add`:
             "meta" "{:doc \"arithmetic addition of 2 arguments\" :arglists ([a b])}"}]}]}
 ```
 
-*Dynamic Metadata*
-
-Pods may send metadata on values returned to the client if metadata encoding is enabled
-for the particular transport format used by the pod.
-
-For example, if your pod uses `:transit+json` as its format, you can enable metadata
-encoding by adding `:transform transit/write-meta` (or whatever transit is aliased to)
-to the optional map passed to `transit/writer`. e.g.:
-
-````clojure
-(transit/writer baos :json {:transform transit/write-meta})
-````
-
-##### From pod client to pod
-
-Currently sending metadata on arguments passed to a pod function is available only for the
-`transit+json` format and can be enabled on a per var basis.
-
-A pod can enable metadata to be read on arguments by sending the "arg-meta" field to "true"
-for the var representing that function. For example:
-
-````clojure
-{:format :transit+json
-    :namespaces [{:name "pod.babashka.demo"
-                  :vars [{"name" "round-trip" "arg-meta" "true"}]}]}
-````
-
 #### Deferred namespace loading
 
 When your pod exposes multiple namespaces that can be used independently from

project.clj

@@ -1,4 +1,4 @@
-(defproject babashka/babashka.pods "0.2.0"
+(defproject babashka/babashka.pods "0.1.0"
   :description "babashka pods"
   :url "https://github.com/babashka/babashka.pods"
   :scm {:name "git"
@@ -8,8 +8,7 @@
   :dependencies [[org.clojure/clojure "1.10.3"]
                  [nrepl/bencode "1.1.0"]
                  [cheshire "5.10.0"]
-                 [babashka/fs "0.1.6"]
+                 [babashka/fs "0.1.6"]]
-                 [com.cognitect/transit-clj "1.0.329"]]
   :deploy-repositories [["clojars" {:url "https://clojars.org/repo"
                                     :username :env/clojars_user
                                     :password :env/clojars_pass

script/changelog.clj

@@ -1,17 +0,0 @@
-#!/usr/bin/env bb
-
-(ns changelog
-  (:require [clojure.string :as str]))
-
-(let [changelog (slurp "CHANGELOG.md")
-      replaced (str/replace changelog
-                            #" #(\d+)"
-                            (fn [[_ issue after]]
-                              (format " [#%s](https://github.com/babashka/pods/issues/%s)%s"
-                                      issue issue (str after))))
-      replaced (str/replace replaced
-                            #"@([a-zA-Z0-9-_]+)([, \.)])"
-                            (fn [[_ name after]]
-                              (format "[@%s](https://github.com/%s)%s"
-                                      name name after)))]
-  (spit "CHANGELOG.md" replaced))

src/babashka/pods/impl.clj

@@ -28,9 +28,6 @@
 (defn bytes->string [^"[B" bytes]
   (String. bytes))
 
-(defn bytes->boolean [^"[B" bytes]
-  (= "true" (String. bytes)))
-
 (defn get-string [m k]
   (-> (get m k)
       bytes->string))
@@ -39,10 +36,6 @@
   (some-> (get m k)
           bytes->string))
 
-(defn get-maybe-boolean [m k]
-  (some-> (get m k)
-          bytes->boolean))
-
 (defn next-id []
   (str (java.util.UUID/randomUUID)))
 
@@ -90,12 +83,10 @@
   (let [wh (transit/write-handler tag-fn val-fn)]
     (swap! transit-default-write-handlers assoc *pod-id* wh)))
 
-(defn transit-json-write
+(defn transit-json-write [pod-id ^String s]
-  [pod-id ^String s metadata?]
   (with-open [baos (java.io.ByteArrayOutputStream. 4096)]
-    (let [w (transit/writer baos :json (cond-> {:handlers (get @transit-write-handler-maps pod-id)
+    (let [w (transit/writer baos :json {:handlers (get @transit-write-handler-maps pod-id)
-                                                :default-handler (get @transit-default-write-handlers pod-id)}
+                                        :default-handler (get @transit-default-write-handlers pod-id)})]
-                                         metadata? (assoc :transform transit/write-meta)))]
       (transit/write w s)
       (str baos))))
 
@@ -107,7 +98,7 @@
         write-fn (case format
                    :edn pr-str
                    :json cheshire/generate-string
-                   :transit+json #(transit-json-write (:pod-id pod) % (:arg-meta opts)))
+                   :transit+json #(transit-json-write (:pod-id pod) %))
         id (next-id)
         chan (if handlers handlers
                  (promise))
@@ -137,12 +128,11 @@
                          edn/read-string)
            name-sym (if vmeta
                       (with-meta name-sym vmeta)
-                      name-sym)
+                      name-sym)]
-           metadata? (get-maybe-boolean var "arg-meta")]
        [name-sym
         (or code
             (fn [& args]
-              (let [res (invoke pod sym args {:async async? :arg-meta metadata?})]
+              (let [res (invoke pod sym args {:async async?})]
                 res)))]))
    vars))
 
@@ -187,17 +177,14 @@
               (let [id (get reply "id")
                     id    (bytes->string id)
                     value* (find reply "value")
-                    [exception value] (try (some->> value*
+                    value (some-> value*
                                   second
                                   bytes->string
-                                                    read-fn
+                                  read-fn)
-                                                    (vector nil))
-                                           (catch Exception e
-                                             [e nil]))
                     status (get reply "status")
                     status (set (map (comp keyword bytes->string) status))
-                    error? (or exception (contains? status :error))
+                    error? (contains? status :error)
-                    done? (or error? exception (contains? status :done))
+                    done? (or error? (contains? status :done))
                     [ex-message ex-data]
                     (when error?
                       [(or (some-> (get reply "ex-message")
@@ -215,9 +202,8 @@
                                    :vars (bencode->vars pod name-str v)}))
                     chan (get @chans id)
                     promise? (instance? clojure.lang.IPending chan)
-                    exception (or exception
+                    exception (when (and promise? error?)
-                                  (when (and promise? error?)
+                                (ex-info ex-message ex-data))
-                                    (ex-info ex-message ex-data)))
                     ;; NOTE: if we need more fine-grained handlers, we will add
                     ;; a :raw handler that will just get the bencode message's raw
                     ;; data
@@ -320,8 +306,7 @@
        (catch java.net.SocketException _ nil)))
 
 (defn port-file [pid]
-  (doto (io/file (str ".babashka-pod-" pid ".port"))
+  (io/file (str ".babashka-pod-" pid ".port")))
-    (.deleteOnExit)))
 
 (defn read-port [^java.io.File port-file]
   (loop []

src/babashka/pods/impl/resolver.clj

@@ -91,10 +91,7 @@
                   ^"[Ljava.nio.file.CopyOption;"
                   (into-array
                    [java.nio.file.StandardCopyOption/REPLACE_EXISTING])))
-    (.mkdirs destination-dir)
+    (sh "tar" "xf" (.getPath tmp-file) "--directory" (.getPath destination-dir))
-    (let [res (sh "tar" "xf" (.getPath tmp-file) "--directory" (.getPath destination-dir))]
-      (when-not (zero? (:exit res))
-        (throw (ex-info (:err res) res))))
     (.delete tmp-file)))
 
 (defn make-executable [dest-dir executables verbose?]
@@ -134,9 +131,13 @@
    "https://raw.githubusercontent.com/babashka/pod-registry/master/manifests/%s/%s/manifest.edn"
    qsym version))
 
+(defn pod-manifest-file
+  [qsym version]
+  (io/file @pods-repo-dir (str qsym) (str version) "manifest.edn"))
+
 (defn pod-manifest
   [qsym version force?]
-  (let [f (io/file @pods-repo-dir (str qsym) (str version) "manifest.edn")]
+  (let [f (pod-manifest-file qsym version)]
     (if (and (not force?)
              (.exists f))
       (edn/read-string (slurp f))
@@ -185,20 +186,16 @@
                  (.digest digest))
         (String. "UTF-8"))))
 
-(defn resolve [qsym version force?]
+(defn install-pod-artifacts
-  (when-not (string? version)
+  [artifacts {cdir :cache-dir, ddir :data-dir, :keys [:force? :pod/options]}]
-    (throw (IllegalArgumentException. "Version must be provided for resolving from pod registry!")))
+  (let [execs (mapv (fn [artifact]
-  (when-let [manifest (pod-manifest qsym version force?)]
-    (let [artifacts (match-artifacts manifest)
-          cdir (cache-dir manifest)
-          ddir (data-dir manifest)
-          execs (mapv (fn [artifact]
                       (let [url (:artifact/url artifact)
                             file-name (last (str/split url #"/"))
                             cache-file (io/file cdir file-name)
-                              executable (io/file ddir (:artifact/executable artifact))]
+                            exe-file-name (:artifact/executable artifact)
+                            executable (io/file ddir exe-file-name)]
                         (when (or force? (not (.exists executable)))
-                            (warn (format "Downloading pod %s (%s)" qsym version))
+                          (warn (format "Downloading pod %s" url))
                           (download url cache-file false)
                           (when-let [expected-sha (:artifact/hash artifact)]
                             (let [sha (sha256 cache-file)]
@@ -217,9 +214,23 @@
                                   (un-tgz cache-file ddir
                                           false))
                             (.delete cache-file))
-                            (make-executable ddir [(:artifact/executable artifact)] false)
+                          (make-executable ddir [exe-file-name] false)
-                            (warn (format "Successfully installed pod %s (%s)" qsym version))
+                          (warn (format "Successfully installed pod %s" exe-file-name))
-                            (io/file ddir (:artifact/executable artifact)))
+                          (io/file ddir exe-file-name))
-                          (io/file ddir (:artifact/executable artifact)))) artifacts)]
+                        (io/file ddir exe-file-name)))
+                    artifacts)]
       {:executable (.getAbsolutePath ^java.io.File (first execs))
-       :options (:pod/options manifest)})))
+       :options options}))
+
+(defn resolve [qsym version force?]
+  (when-not (string? version)
+    (throw (IllegalArgumentException. "Version must be provided for resolving from pod registry!")))
+  (when-let [manifest (pod-manifest qsym version force?)]
+    (let [cdir (cache-dir manifest)
+          ddir (data-dir manifest)
+          artifacts (match-artifacts manifest)
+          pod-options (:pod/options manifest)]
+      (install-pod-artifacts artifacts {:cache-dir cdir
+                                        :data-dir ddir
+                                        :force? force?
+                                        :pod/options pod-options}))))

src/babashka/pods/jvm.clj

@@ -12,12 +12,11 @@
 (defn- process-namespace [{:keys [:name :vars]}]
   (binding [*ns* (load-string (format "(ns %s) *ns*" name))]
     (doseq [[var-sym v] vars]
-      (when-let [maybe-core (some-> (ns-resolve *ns* var-sym) meta :ns str symbol)]
-        (when (= 'clojure.core maybe-core)
-          (ns-unmap *ns* var-sym)))
       (cond
         (ifn? v)
-        (intern name var-sym v)
+        (do
+          (ns-unmap *ns* var-sym)
+          (intern name var-sym v))
         (string? v)
         (load-string v)))))
 

src/babashka/pods/sci.clj

@@ -41,17 +41,32 @@
       (with-open [r (PushbackInputStream. (io/input-stream cache-file))]
         (impl/read r)))))
 
-(defn load-pod-metadata* [bb-edn-file pod-spec {:keys [:version :cache] :as opts}]
+(defn cache-pod-metadata! [metadata {:keys [:bb-edn-file :pod-name :pod-version]}]
-  (let [metadata (impl/load-pod-metadata pod-spec opts)
+  (let [cache-file (metadata-cache-file bb-edn-file pod-name
-        cache-file (when (and metadata cache)
+                                        {:version pod-version})]
-                     (metadata-cache-file bb-edn-file pod-spec opts))]
     (when cache-file
       (io/make-parents cache-file)
       (when (fs/writable? (fs/parent cache-file))
         (with-open [w (io/output-stream cache-file)]
-          (impl/write w metadata))))
+          (impl/write w metadata))))))
+
+(defn load-pod-metadata* [bb-edn-file pod-spec {:keys [:version :cache] :as opts}]
+  (let [metadata (impl/load-pod-metadata pod-spec opts)]
+    (when (and metadata cache)
+      (cache-pod-metadata! metadata {:bb-edn-file bb-edn-file
+                                     :pod-name pod-spec
+                                     :pod-version version}))
     metadata))
 
+(defn pod-namespaces
+  [pod-name metadata opts]
+  (reduce
+    (fn [pod-namespaces ns]
+      (let [ns-sym (-> ns (get "name") impl/bytes->string symbol)]
+        (assoc pod-namespaces ns-sym {:pod-spec pod-name
+                                      :opts (assoc opts :metadata metadata)})))
+    {} (get metadata "namespaces")))
+
 (defn load-pod-metadata
   ([pod-spec opts] (load-pod-metadata nil pod-spec opts))
   ([bb-edn-file pod-spec {:keys [:cache] :as opts}]
@@ -62,12 +77,7 @@
                                                               opts))]
            cached-metadata
            (load-pod-metadata* bb-edn-file pod-spec opts))]
-     (reduce
+     (pod-namespaces pod-spec metadata opts))))
-       (fn [pod-namespaces ns]
-         (let [ns-sym (-> ns (get "name") impl/bytes->string symbol)]
-           (assoc pod-namespaces ns-sym {:pod-spec pod-spec
-                                         :opts (assoc opts :metadata metadata)})))
-       {} (get metadata "namespaces")))))
 
 (defn load-pod
   ([ctx pod-spec] (load-pod ctx pod-spec nil))
@@ -122,6 +132,29 @@
      (sci/future (impl/processor pod))
      {:pod/id (:pod-id pod)})))
 
+(defn load-pod-metadata-from-manifest
+  [manifest {:keys [:bb-edn-file]}]
+  (let [artifacts (resolver/match-artifacts manifest)
+        pod-name (:pod/name manifest)]
+    (when artifacts
+      (let [cdir (resolver/cache-dir manifest)
+            ddir (resolver/data-dir manifest)
+            pod (resolver/install-pod-artifacts
+                  artifacts {:cache-dir cdir
+                             :data-dir ddir
+                             :pod/options (:pod/options manifest)})
+            metadata (impl/run-pod-for-metadata [(:executable pod)] nil)]
+        (let [pod-version (:pod/version manifest)
+              opts {:bb-edn-file bb-edn-file
+                    :pod-name pod-name
+                    :pod-version pod-version}]
+          (cache-pod-metadata! metadata opts)
+          (pod-namespaces pod-name metadata {:version pod-version}))))))
+
+(defn pod-manifest-file
+  [manifest]
+  (resolver/pod-manifest-file (-> manifest :pod/name symbol) (:pod/version manifest)))
+
 (defn unload-pod
   ([pod-id] (unload-pod pod-id {}))
   ([pod-id _opts]

test-pod/pod/test_pod.clj

@@ -61,12 +61,6 @@
       (transit/write w s)
       (str baos))))
 
-(defn transit-json-write-meta [s]
-  (with-open [baos (java.io.ByteArrayOutputStream. 4096)]
-    (let [w (transit/writer baos :json {:transform transit/write-meta})]
-      (transit/write w s)
-      (str baos))))
-
 (defn run-pod [cli-args]
   (let [format (cond (contains? cli-args "--json") :json
                      (contains? cli-args "--transit+json") :transit+json
@@ -136,10 +130,6 @@
                                                 {"name" "read-other-tag"
                                                  "code" "(defn read-other-tag [x] [x x])"
                                                  "meta" "{:doc \"unread\"}"}
-                                                {"name" "round-trip-meta"
-                                                 "arg-meta" "true"}
-                                                {"name" "dont-round-trip-meta"
-                                                 "arg-meta" "false"}
                                                 {"name" "-local-date-time"}
                                                 {"name" "transit-stuff"
                                                  "code" "
@@ -161,8 +151,7 @@
 (babashka.pods/add-transit-read-handler! \"java.array\"
   into-array)
 
-"}
+"}]
-                                                {"name" "incorrect-edn"}]
                                                dependents)}
                                  {"name" "pod.test-pod.loaded"
                                   "defer" "true"}
@@ -245,32 +234,11 @@
                              {"status" ["done"]
                               "id" id
                               "value" "#my/other-tag[1]"})
-                            pod.test-pod/round-trip-meta
-                            (write out
-                                   {"status" ["done"]
-                                    "id" id
-                                    "value"
-                                    (case format
-                                      :transit+json (transit-json-write-meta (first args))
-                                      (write-fn (first args)))})
-                            pod.test-pod/dont-round-trip-meta
-                            (write out
-                                   {"status" ["done"]
-                                    "id" id
-                                    "value"
-                                    (case format
-                                      :transit+json (transit-json-write-meta (first args))
-                                      (write-fn (first args)))})
                             pod.test-pod/-local-date-time
                             (write out
                                    {"status" ["done"]
                                     "id" id
-                                    "value" (write-fn (first args))})
+                                    "value" (write-fn (first args))}))
-                            pod.test-pod/incorrect-edn
-                            (write out
-                                   {"status" ["done"]
-                                    "id" id
-                                    "value" (write-fn {(keyword "foo bar") 1})}))
                           (recur))
                 :shutdown (System/exit 0)
                 :load-ns (let [ns (-> (get message "ns")

test-resources/test_program.clj

@@ -84,31 +84,12 @@
       (.isArray (class v)))
     true))
 
-(def round-trip-meta
-  (if (= "transit+json" fmt)
-    (= {:my-meta 2} (meta (pod.test-pod/round-trip-meta (with-meta [2] {:my-meta 2}))))
-    true))
-
-(def round-trip-meta-nested
-  (if (= "transit+json" fmt)
-    (= {:my-meta 3} (meta (first (pod.test-pod/round-trip-meta [(with-meta [3] {:my-meta 3})]))))
-    true))
-
-(def dont-round-trip-meta
-  (if (= "transit+json" fmt)
-    (= nil (meta (pod.test-pod/dont-round-trip-meta (with-meta [2] {:my-meta 2}))))
-    true))
-
 (require '[pod.test-pod.only-code :as only-code])
 (def should-be-1 (only-code/foo))
 
 (require '[pod.test-pod.loaded2 :as loaded2])
 (def loaded (loaded2/loaded 1))
 
-(def incorrect-edn-response
-  (try (pod.test-pod/incorrect-edn)
-       (catch Exception e (ex-message e))))
-
 (pods/unload-pod pod-id)
 (def successfully-removed (nil? (find-ns 'pod.test-pod)))
 
@@ -131,11 +112,7 @@
  fn-called
  local-date-time
  assoc-string-array
- round-trip-meta
- round-trip-meta-nested
- dont-round-trip-meta
  should-be-1
  add-sync-meta
  error-meta
- read-other-tag-meta
+ read-other-tag-meta]
- incorrect-edn-response]

test/babashka/pods/sci_test.clj

@@ -23,9 +23,7 @@
         _ (vreset! ctx-ref ctx)
         ret (sci/binding [sci/out out
                           sci/err err]
-              (binding [*out* out
+              (sci/eval-string* ctx test-program))]
-                        *err* err]
-                (sci/eval-string* ctx test-program)))]
     (assertions out err ret)))
 
 (deftest pod-registry-test

test/babashka/pods/test_common.clj

@@ -1,6 +1,5 @@
 (ns babashka.pods.test-common
   (:require [clojure.java.io :as io]
-            [clojure.string :as str]
             [clojure.test :refer [is]]))
 
 (def test-program (slurp (io/file "test-resources" "test_program.clj")))
@@ -10,12 +9,7 @@
   ;; (.println System/err out)
   ;; (.println System/err err)
   (doseq [[expected actual]
-          (map vector (replace
+          (map vector '["pod.test-pod"
-                       {::edn-error (if (= "edn"
-                                           (System/getenv "BABASHKA_POD_TEST_FORMAT"))
-                                      "Map literal must contain an even number of forms"
-                                      ::dont-care)}
-                       '["pod.test-pod"
                         pod.test-pod
                         {:a 1, :b 2}
                         6
@@ -34,21 +28,15 @@
                         3
                         true ;; local-date
                         true ;; roundtrip string array
-                         true ;; roundtrip metadata
-                         true ;; roundtrip metadata nested
-                         true ;; dont roundtrip metadata (when arg-meta "false"/ absent)
                         1
                         "add the arguments"
                         nil
-                         nil
+                        nil]
-                         ::edn-error])
                (concat ret (repeat ::nil)))]
-    (cond (instance? java.util.regex.Pattern expected)
+    (if (instance? java.util.regex.Pattern expected)
       (is (re-find expected actual))
-          (= ::dont-care expected) nil
-          :else
       (is (= expected actual))))
   (is (= "(\"hello\" \"print\" \"this\" \"debugging\" \"message\")\n:foo\n:foo\n" (str out)))
-  (is (str/starts-with? (str err) "(\"hello\" \"print\" \"this\" \"error\")" )))
+  (is (= "(\"hello\" \"print\" \"this\" \"error\")\n" (str err))))
 
 (def pod-registry (slurp (io/file "test-resources" "pod_registry.clj")))