Fix tests

Still need to make this work w/ :local/root pod libs and in uberscripts

CHANGELOG.md

@@ -10,6 +10,7 @@ A preview of the next release can be installed from
 - Add single argument read method support to PipedInputStream proxy ([@retrogradeorbit](https://github.com/retrogradeorbit))
 - feat: Honor `*print-namespace-maps*` in pprint ([@ghoseb](https://github.com/ghoseb))
 - [#1369](https://github.com/babashka/babashka/issues/1369): provide `.sha256` files for every released asset
+- [#1390](https://github.com/babashka/babashka/pull/1390): Add support for distributing pods as Clojure libraries containing just the manifest.
 
 ## 0.10.163 (2022-09-24)
 

deps.edn

@@ -27,7 +27,7 @@
         babashka/babashka.core {:local/root "babashka.core"}
         borkdude/graal.locking {:mvn/version "0.0.2"},
         org.clojure/core.async {:mvn/version "1.5.648"},
-        org.clojure/tools.cli {:mvn/version "1.0.206"},
+        org.clojure/tools.cli {:mvn/version "1.0.214"},
         org.clojure/data.csv {:mvn/version "1.0.0"},
         cheshire/cheshire {:mvn/version "5.11.0"}
         org.clojure/data.xml {:mvn/version "0.2.0-alpha8"}
@@ -50,7 +50,7 @@
         org.clojure/data.priority-map {:mvn/version "1.1.0"}
         insn/insn {:mvn/version "0.5.2"}
         org.clojure/core.rrb-vector {:mvn/version "0.1.2"}
-        org.babashka/cli {:mvn/version "0.4.38"}}
+        org.babashka/cli {:mvn/version "0.5.40"}}
  :aliases {:babashka/dev
            {:main-opts ["-m" "babashka.main"]}
            :profile
@@ -154,7 +154,9 @@
                                           :deps/manifest :deps}
                          exoscale/interceptor {:git/url "https://github.com/exoscale/interceptor"
                                                :git/sha "ca115fe00a0abf3a2f78452ab309c3aa4c00fc4e"
-                                               :deps/manifest :deps}}
+                                               :deps/manifest :deps}
+                         lambdaisland/uri {:git/url "https://github.com/lambdaisland/uri"
+                                           :git/sha "ac4f1f9c8e4f45a088db1c6383ce2191c973987c"}}
             :classpath-overrides {org.clojure/clojure nil
                                   org.clojure/spec.alpha nil}}
            :clj-nvd

doc/pod-libraries.md

@@ -0,0 +1,113 @@
+# Pods distributed as manifest libraries
+
+Babashka library pods are a way of distributing pods that uses the normal
+Clojure library dependency system to install and load pods.
+It eliminates the need for the centralized babashka pod registry because the
+pod manifest is (the only thing) downloaded in the library JAR file.
+This document assumes you are already familiar with Babashka pods themselves
+and only covers distributing them via Clojure libraries.
+
+Let's break down the various components of how this works:
+
+1. The `pod-manifest.edn` file
+    1. The first thing your pod needs is a `pod-manifest.edn` file. It is
+       recommended to store this in your pod project's `resources` dir under a
+       directory hierarchy matching your pod's top-level exposed namespace (but
+       replacing hyphens with underscores as usual in Clojure → Java name
+       munging). So, for example, if your pod exposes namespaces starting with
+       `pod.foo`, you should put your `pod-manifest.edn` in this path in your
+       pod project: `resources/pod/foo/pod-manifest.edn`.
+        1. By putting your `pod-manifest.edn` file in this namespace-mirroring
+           directory hierarchy inside your `resources` dir, you can consume it
+           while developing your pod using tools.deps' handy `:local/root`
+           feature.
+        2. If you have more than one pod in the same project (or just prefer
+           it), you can nest the `pod-manifest.edn` files deeper into their
+           respective namespace hierarchies. As long as there are at least two
+           namespace element directories, babashka will find the corresponding
+           `pod-manifest.edn` at runtime. So, for example, you can have
+           `foo.bar.baz.qux` → `resources/foo/bar/baz/qux/pod-manifest.edn` but
+           not just `foo` → `resources/foo/pod-manifest.edn`. This is because
+           a) single-element namespaces are frowned upon anyway in Clojure and
+           b) the first namespace element is often just `com` or `pod` or
+           similar and `pod/pod-manifest.edn` would be a very ambiguous
+           resource path to look for in a typical classpath.
+    2. Make sure the `resources` dir is in the `:paths` vector in your pod
+       project's `deps.edn` file. E.g. `:paths ["src" "resources"]`.
+    3. The format of the `pod-manifest.edn` file looks like this:
+
+```clojure
+{:pod/name pod.foo/bar
+ :pod/description "Foo bar pod"
+ :pod/version "0.4.0"
+ :pod/license "MIT"
+ :pod/artifacts
+ [{:os/name "Linux.*"
+   :os/arch "amd64"
+   :artifact/url "https://github.com/foo/pod-bar/releases/download/v0.4.0/pod-foo-bar-linux-amd64.zip"
+   :artifact/executable "pod-foo-bar"}
+  {:os/name "Linux.*"
+   :os/arch "aarch64"
+   :artifact/url "https://github.com/foo/pod-bar/releases/download/v0.4.0/pod-foo-bar-linux-arm64.zip"
+   :artifact/executable "pod-foo-bar"}
+  {:os/name "Mac.*"
+   :os/arch "amd64"
+   :artifact/url "https://github.com/foo/pod-bar/releases/download/v0.4.0/pod-foo-bar-macos-amd64.zip"
+   :artifact/executable "pod-foo-bar"}]}
+```
+
+1. The library JAR file
+    1. This JAR file can be built however you want (but see below for an example
+       tools.build `build.clj` snippet for generating it), but it must contain a
+       `pod-manifest.edn` file in a path corresponding to your pod's top-level
+       namespace as described above. For example, if your pod exposes the
+       namespaces `pod.foo.bar.baz` & `pod.foo.bar.qux` then you should have this
+       directory layout inside your JAR file: `pod/foo/bar/pod-manifest.edn`.
+    2. You then distribute this library JAR to any Maven repo you like
+       (e.g. Clojars).
+2. The consuming project
+    1. In the project that wants to use your `pod.foo/bar` pod, you add the
+       library's Maven coordinates to your `deps.edn` or `bb.edn` `:deps`
+       section. This will put the `pod/foo/bar/pod-manifest.edn` file onto your
+       project's classpath. `bb` will then be able to find your pod's manifest
+       and install and load the pod.
+3. Example `build.clj` snippet for building the library JAR
+
+```clojure
+(ns build
+  (:require
+    [clojure.string :as str]
+    [clojure.tools.build.api :as b]))
+
+(def lib 'pod.foo/bar)
+(def version "0.4.0")
+(def jar-sources "resources")
+(def class-dir "target/resources")
+(def basis (b/create-basis {:project "deps.edn"}))
+(def pod-manifest "pod/foo/bar/pod-manifest.edn")
+(def jar-file (format "target/%s-%s.jar" (-> lib namespace (str/replace "." "-")) (name lib)))
+
+(defn clean [_]
+  (b/delete {:path "target"}))
+
+(defn jar [_]
+  (b/write-pom {:class-dir class-dir
+                :lib       lib
+                :version   version
+                :basis     basis})
+  (b/copy-file {:src    (str jar-sources "/" pod-manifest)
+                :target (str class-dir "/" pod-manifest)})
+  (b/jar {:class-dir class-dir
+          :jar-file  jar-file}))
+
+;; optional - allows you to install the JAR to your local ~/.m2/repository for testing
+(defn install [_]
+  (jar nil)
+  (b/install {:basis     basis
+              :lib       lib
+              :version   version
+              :jar-file  jar-file
+              :class-dir class-dir}))
+```
+
+You can then build this pod library via: `clj -T:build jar` and the JAR file will be in `target/pod-foo-bar.jar`.

feature-hiccup/babashka/impl/hiccup.clj

@@ -55,7 +55,8 @@
   {'html (copy-var html-1 hns {:name 'html})})
 
 (def hiccup2-namespace
-  {'html (copy-var html-2 hns2 {:name 'html})})
+  {'html (copy-var html-2 hns2 {:name 'html})
+   'raw (copy-var util/raw-string hns2 {:name 'raw})})
 
 (def html-mode (copy-var util/*html-mode* uns))
 (def escape-strings? (copy-var util/*escape-strings?* uns))
@@ -63,7 +64,8 @@
 (def hiccup-util-namespace
   {'*html-mode* html-mode
    '*escape-strings?* escape-strings?
-   'raw-string (copy-var util/raw-string uns)})
+   'raw-string (copy-var util/raw-string uns)
+   'to-uri (copy-var util/to-uri uns)})
 
 (defn render-html [& contents]
   (binding [util/*html-mode* @html-mode

fs

@@ -1 +1 @@
-Subproject commit 73210e83808b8bfe664cba7d2148464a2fa38aae
+Subproject commit 75bdf5aa42a8a34e4e38279f79d761887dc46847

pods

@@ -1 +1 @@
-Subproject commit decf791000081ca9e6d2fbea9f20a0aa3fae902e
+Subproject commit 951a6297d18da7d0ee60def06812141e0ce7fbb2

project.clj

@@ -21,7 +21,7 @@
   :dependencies [[org.clojure/clojure "1.11.1"]
                  [borkdude/edamame "1.0.0"]
                  [borkdude/graal.locking "0.0.2"]
-                 [org.clojure/tools.cli "1.0.206"]
+                 [org.clojure/tools.cli "1.0.214"]
                  [cheshire "5.11.0"]
                  [nrepl/bencode "1.1.0"]
                  [borkdude/sci.impl.reflector "0.0.1"]
@@ -32,7 +32,7 @@
                  [com.github.clj-easy/graal-build-time "0.1.0"]
                  [rewrite-clj/rewrite-clj "1.1.45"]
                  [insn/insn "0.5.2"]
-                 [org.babashka/cli "0.4.38"]]
+                 [org.babashka/cli "0.5.40"]]
   :plugins       [[org.kipz/lein-meta-bom "0.1.1"]]
   :metabom {:jar-name "metabom.jar"}
   :profiles {:feature/xml  {:source-paths ["feature-xml"]

src/babashka/impl/classpath.clj

@@ -5,7 +5,8 @@
             [clojure.java.io :as io]
             [clojure.string :as str]
             [sci.core :as sci])
-  (:import [java.util.jar JarFile Manifest]
+  (:import (java.io File)
+           [java.util.jar JarFile Manifest]
            (java.net URL)))
 
 (set! *warn-on-reflection* true)
@@ -23,19 +24,19 @@
          (when (.exists f)
            (if url?
              ;; manual conversion, faster than going through .toURI
-             (java.net.URL. "file" nil (.getAbsolutePath f))
+             (URL. "file" nil (.getAbsolutePath f))
              {:file (.getAbsolutePath f)
               :source (slurp f)}))))
      resource-paths)))
 
 (defn path-from-jar
-  [^java.io.File jar-file resource-paths url?]
+  [^File jar-file resource-paths url?]
   (with-open [jar (JarFile. jar-file)]
     (some (fn [path]
             (when-let [entry (.getEntry jar path)]
               (if url?
                 ;; manual conversion, faster than going through .toURI
-                (java.net.URL. "jar" nil
+                (URL. "jar" nil
                       (str "file:" (.getAbsolutePath jar-file) "!/" path))
                 {:file path
                  :source (slurp (.getInputStream jar entry))})))
@@ -61,19 +62,36 @@
 
 (def path-sep (System/getProperty "path.separator"))
 
-(defn loader [^String classpath]
-  (let [parts (.split classpath path-sep)
-        entries (keep part->entry parts)]
-    (Loader. entries)))
+(defn classpath-entries [^String classpath]
+  (let [parts (.split classpath path-sep)]
+    (keep part->entry parts)))
 
-(defn source-for-namespace [loader namespace opts]
+(defn loader [^String classpath]
+    (Loader. (classpath-entries classpath)))
+
+(declare get-classpath)
+
+(defn source-for-namespace
+  ([namespace opts]
+   (some-> (get-classpath) loader (source-for-namespace namespace opts)))
+  ([loader namespace opts]
    (let [ns-str (name namespace)
          ^String ns-str (munge ns-str)
          ;; do NOT pick the platform specific file separator here, since that doesn't work for searching in .jar files
          ;; (io/file "foo" "bar/baz") does work on Windows, despite the forward slash
          base-path (.replace ns-str "." "/")
-        resource-paths (mapv #(str base-path %) [".bb" ".clj" ".cljc"])]
-    (getResource loader resource-paths opts)))
+         manifest-paths (loop [ns (str/split ns-str #"\.")
+                               paths []]
+                          (let [path (str/join "/" (conj ns "pod-manifest.edn"))
+                                next-ns (-> ns butlast vec)
+                                next-paths (conj paths path)]
+                            (if (< 1 (count next-ns)) ; don't look in top-level (e.g. com, pod) namespaces
+                              (recur next-ns next-paths)
+                              next-paths)))
+         resource-paths   (into (mapv #(str base-path %)
+                                      [".bb" ".clj" ".cljc"])
+                                manifest-paths)]
+     (getResource loader resource-paths opts))))
 
 (defn main-ns [manifest-resource]
   (with-open [is (io/input-stream manifest-resource)]

src/babashka/impl/pods.clj

@@ -32,10 +32,19 @@
                              "for pods on your local filesystem."))))))
     {} pods-map))
 
+(defn load-pod-metadata-from-manifest
+  [manifest]
+  (pods/load-pod-metadata-from-manifest manifest (-> @bb-edn :file io/file)))
+
+(defn pod-manifest-file
+  [manifest]
+  (pods/pod-manifest-file manifest))
+
 (def podns (sci/create-ns 'babashka.pods nil))
 
 (def pods-namespace
   {'load-pod (sci/copy-var load-pod podns)
+   'load-pod-metadata-from-manifest (sci/copy-var load-pod-metadata-from-manifest podns)
    'invoke (sci/copy-var pods/invoke podns)
    'unload-pod (sci/copy-var pods/unload-pod podns)
    'add-transit-read-handler! (sci/copy-var pods/add-transit-read-handler! podns)

src/babashka/main.clj

@@ -408,14 +408,14 @@ Use bb run --help to show this help output.
                                                          (sci/create-ns 'clojure.core.rrb-vector))})
        'edamame.core edamame-namespace
        'sci.core {'format-stacktrace (sci/copy-var sci/format-stacktrace sci-ns)
-                  'stacktrace (sci/copy-var sci/stacktrace sci-ns)
+                  'stacktrace (sci/copy-var sci/stacktrace sci-ns)}
                   ;; 'eval-string (sci/copy-var sci/eval-string sci-ns)
                   ;; 'eval-string* (sci/copy-var sci/eval-string* sci-ns)
                   ;; 'init (sci/copy-var sci/init sci-ns)
                   ;; 'fork (sci/copy-var sci/fork sci-ns)
-                  }
-       'babashka.cli cli/cli-namespace
-       }
+
+       'babashka.cli cli/cli-namespace}
+
     features/xml?  (assoc 'clojure.data.xml @(resolve 'babashka.impl.xml/xml-namespace)
                           'clojure.data.xml.event @(resolve 'babashka.impl.xml/xml-event-namespace)
                           'clojure.data.xml.tree @(resolve 'babashka.impl.xml/xml-tree-namespace))
@@ -767,6 +767,70 @@ Use bb run --help to show this help output.
       env-os-name-present? (not= env-os-name sys-os-name)
       env-os-arch-present? (not= env-os-arch sys-os-arch))))
 
+(def namespace-warnings
+  {'clojure.spec.alpha "[babashka] WARNING: Use the babashka-compatible version of clojure.spec.alpha, available here: https://github.com/babashka/spec.alpha"
+   'clojure.core.specs.alpha "[babashka] WARNING: clojure.core.specs.alpha is removed from the classpath, unless you explicitly add the dependency."})
+
+(defn build-uberscript-load-fn
+  [sources]
+  (fn [{:keys [:namespace :reload]}]
+    (let [{:keys [:loader]} @cp/cp-state]
+      (or
+        (when (not reload)
+          (if (or (contains? namespaces namespace)
+                  (contains? sci-namespaces/namespaces namespace))
+            "" ;; ignore built-in namespaces
+            (when-let [{:keys [:pod-spec :opts]} (get @pod-namespaces namespace)]
+              (swap! sources conj
+                     (format
+                       "(babashka.pods/load-pod '%s \"%s\" '%s)\n"
+                       pod-spec (:version opts)
+                       (dissoc opts :version :metadata)))
+              {})))
+        (when (contains? namespace-warnings namespace)
+          (binding [*out* *err*]
+            (println (get namespace-warnings namespace)))
+          nil)
+        (when loader
+          (when-let [{:keys [:file :source]} (cp/source-for-namespace loader namespace nil)]
+            (let [src (if (str/ends-with? file "/pod-manifest.edn")
+                        (format
+                          "(babashka.pods/load-pod-metadata-from-manifest '%s)\n"
+                          source)
+                        source)]
+              (swap! sources conj src)
+              (uberscript/uberscript {:ctx @common/ctx
+                                      :expressions [src]})
+              {})))))))
+
+(defn build-load-fn
+  []
+  (fn [{:keys [:namespace :reload]}]
+    (let [{:keys [loader]} @cp/cp-state]
+      (or
+        ;; pod namespaces go before namespaces from source,
+        ;; unless reload is used
+        (when-not reload
+          (when-let [pod (get @pod-namespaces namespace)]
+            (pods/load-pod (:pod-spec pod) (:opts pod))
+            {}))
+        (when (contains? namespace-warnings namespace)
+          (binding [*out* *err*]
+            (println (get namespace-warnings namespace)))
+          nil)
+        (when loader
+          (when-let [{:keys [:file :source] :as res} (cp/source-for-namespace loader namespace nil)]
+            (if (str/ends-with? file "/pod-manifest.edn")
+              (let [manifest (edn/read-string source)]
+                (when-let [pod-nses (pods/load-pod-metadata-from-manifest manifest)]
+                  ;; TODO: Make the pod loading code look for the manifest in the lib instead?
+                  (spit (pods/pod-manifest-file manifest) source)
+                  (vswap! pod-namespaces merge pod-nses)
+                  (let [pod (get pod-nses namespace)]
+                    (pods/load-pod (:pod-spec pod) (:opts pod)))
+                  {}))
+              res)))))))
+
 (defn exec [cli-opts]
   (binding [*unrestricted* true]
     (sci/binding [core/warn-on-reflection @core/warn-on-reflection
@@ -820,46 +884,9 @@ Use bb run --help to show this help output.
                          abs-path))
             _ (when jar
                 (cp/add-classpath jar))
-            load-fn (fn [{:keys [:namespace :reload]}]
-                      (let [{:keys [loader]}
-                            @cp/cp-state]
-                        (or
-                         (when ;; ignore built-in namespaces when uberscripting, unless with :reload
-                             (and uberscript
-                                  (not reload)
-                                  (or (contains? namespaces namespace)
-                                      (contains? sci-namespaces/namespaces namespace)))
-                           "")
-                         ;; pod namespaces go before namespaces from source,
-                         ;; unless reload is used
-                         (when-not reload
-                           (when-let [pod (get @pod-namespaces namespace)]
-                             (if uberscript
-                               (do
-                                 (swap! uberscript-sources conj
-                                        (format
-                                         "(babashka.pods/load-pod '%s \"%s\" '%s)\n"
-                                         (:pod-spec pod) (:version (:opts pod))
-                                         (dissoc (:opts pod)
-                                                 :version :metadata)))
-                                 {})
-                               (pods/load-pod (:pod-spec pod) (:opts pod)))))
-                         (when loader
-                           (when-let [res (cp/source-for-namespace loader namespace nil)]
-                             (if uberscript
-                               (do (swap! uberscript-sources conj (:source res))
-                                   (uberscript/uberscript {:ctx @common/ctx
-                                                           :expressions [(:source res)]})
-                                   {})
-                               res)))
-                         (case namespace
-                           clojure.spec.alpha
-                           (binding [*out* *err*]
-                             (println "[babashka] WARNING: Use the babashka-compatible version of clojure.spec.alpha, available here: https://github.com/babashka/spec.alpha"))
-                           clojure.core.specs.alpha
-                           (binding [*out* *err*]
-                             (println "[babashka] WARNING: clojure.core.specs.alpha is removed from the classpath, unless you explicitly add the dependency."))
-                           nil))))
+            load-fn (if uberscript
+                      (build-uberscript-load-fn uberscript-sources)
+                      (build-load-fn))
             main (if (and jar (not main))
                    (when-let [res (cp/getResource
                                    (cp/loader jar)
@@ -887,7 +914,7 @@ Use bb run --help to show this help output.
             _ (when-let [pods (:pods @common/bb-edn)]
                 (when-let [pod-metadata (pods/load-pods-metadata
                                           pods {:download-only (download-only?)})]
-                  (vreset! pod-namespaces pod-metadata)))
+                  (vswap! pod-namespaces merge pod-metadata)))
             preloads (some-> (System/getenv "BABASHKA_PRELOADS") (str/trim))
             [expressions exit-code]
             (cond expressions [expressions nil]