feat #863: Load pods from bb.edn (#1205)

Co-authored-by: Michiel Borkent <michielborkent@gmail.com>

.circleci/config.yml

@@ -14,7 +14,7 @@ commands:
 jobs:
   jvm:
     docker:
-      - image: circleci/clojure:openjdk-11-lein-2.9.6-bullseye
+      - image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye
     working_directory: ~/repo
     environment:
       LEIN_ROOT: "true"
@@ -68,7 +68,7 @@ jobs:
           key: v1-dependencies-{{ checksum "project.clj" }}-{{ checksum "deps.edn" }}
   linux:
     docker:
-      - image: circleci/clojure:openjdk-11-lein-2.9.6-bullseye
+      - image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye
     working_directory: ~/repo
     environment:
       LEIN_ROOT: "true"
@@ -134,7 +134,7 @@ jobs:
             ./bb .circleci/script/publish_artifact.clj || true
   linux-static:
     docker:
-      - image: circleci/clojure:openjdk-11-lein-2.9.6-bullseye
+      - image: circleci/clojure:openjdk-11-lein-2.9.8-bullseye
     working_directory: ~/repo
     environment:
       LEIN_ROOT: "true"
@@ -422,7 +422,7 @@ jobs:
   deploy:
     resource_class: large
     docker:
-      - image: circleci/clojure:lein-2.9.6
+      - image: circleci/clojure:lein-2.9.8
     working_directory: ~/repo
     environment:
       LEIN_ROOT: "true"

CHANGELOG.md

@@ -5,6 +5,13 @@ For a list of breaking changes, check [here](#breaking-changes).
 A preview of the next release can be installed from
 [babashka-dev-builds](https://github.com/babashka/babashka-dev-builds).
 
+## 0.8.0 (TBD)
+
+### New
+
+- [#863](https://github.com/babashka/babashka/issues/863): allow pods to be declared in `bb.edn` and load them when required
+  - See [updated pod library docs for details](TODO: link)
+
 ## 0.7.8 (2022-03-13)
 
 This release improves compatibility with several libraries: [loom](https://github.com/aysylu/loom), [hugsql.core](https://www.hugsql.org) and [specter](https://github.com/redplanetlabs/specter)!

README.md

@@ -272,7 +272,7 @@ Have a library that is already compatible or has recently been made compatible?
 
 Pods are programs that can be used as a Clojure library by
 babashka. Documentation is available in the [pod library
-repo](https://github.com/babashka/babashka.pods).
+repo](https://github.com/babashka/pods).
 
 A list of available pods can be found [here](doc/projects.md#pods).
 

deps.edn

@@ -10,6 +10,7 @@
          "feature-selmer"
          "feature-logging"
          "feature-priority-map"
+         "feature-jdbc"
          "pods/src"
          "babashka.nrepl/src"
          "depstar/src" "process/src"
@@ -137,9 +138,12 @@
            {:extra-deps {clj-nvd/clj-nvd {:git/url "https://github.com/miikka/clj-nvd.git"
                                           :sha "f2ec98699e057a379baf170cb49cf7ad76874a70"}}
             :main-opts ["-m" "clj-nvd.core"]}
- :test ;; added by neil
- {:extra-paths ["test"]
-  :extra-deps {io.github.cognitect-labs/test-runner
-                {:git/tag "v0.5.0" :git/sha "b3fd0d2"}}
-  :main-opts ["-m" "cognitect.test-runner"]
-  :exec-fn cognitect.test-runner.api/test}}}
+           :test
+           {:extra-paths ["test"]
+            :extra-deps {io.github.cognitect-labs/test-runner
+                          {:git/tag "v0.5.0" :git/sha "b3fd0d2"}}
+            :main-opts ["-m" "cognitect.test-runner"]
+            :exec-fn cognitect.test-runner.api/test}
+           :test-pod
+           {:extra-paths ["test-resources"]
+            :main-opts ["-m" "babashka.main" "test-resources/pod.clj"]}}}

resources/META-INF/babashka/deps.edn

@@ -10,6 +10,7 @@
          "feature-selmer"
          "feature-logging"
          "feature-priority-map"
+         "feature-jdbc"
          "pods/src"
          "babashka.nrepl/src"
          "depstar/src" "process/src"
@@ -137,9 +138,12 @@
            {:extra-deps {clj-nvd/clj-nvd {:git/url "https://github.com/miikka/clj-nvd.git"
                                           :sha "f2ec98699e057a379baf170cb49cf7ad76874a70"}}
             :main-opts ["-m" "clj-nvd.core"]}
- :test ;; added by neil
- {:extra-paths ["test"]
-  :extra-deps {io.github.cognitect-labs/test-runner
-                {:git/tag "v0.5.0" :git/sha "b3fd0d2"}}
-  :main-opts ["-m" "cognitect.test-runner"]
-  :exec-fn cognitect.test-runner.api/test}}}
+           :test
+           {:extra-paths ["test"]
+            :extra-deps {io.github.cognitect-labs/test-runner
+                          {:git/tag "v0.5.0" :git/sha "b3fd0d2"}}
+            :main-opts ["-m" "cognitect.test-runner"]
+            :exec-fn cognitect.test-runner.api/test}
+           :test-pod
+           {:extra-paths ["test-resources"]
+            :main-opts ["-m" "babashka.main" "test-resources/pod.clj"]}}}

src/babashka/impl/pods.clj

@@ -1,13 +1,37 @@
 (ns babashka.impl.pods
   {:no-doc true}
   (:refer-clojure :exclude [read])
-  (:require [babashka.impl.common :refer [ctx]]
+  (:require [babashka.impl.common :refer [ctx bb-edn]]
             [babashka.pods.sci :as pods]
-            [sci.core :as sci]))
+            [sci.core :as sci]
+            [clojure.java.io :as io]))
 
 (defn load-pod [& args]
   (apply pods/load-pod @ctx args))
 
+(defn load-pods-metadata [pods-map]
+  (reduce-kv
+    (fn [pod-namespaces pod-spec coord]
+      (merge pod-namespaces
+             (condp #(contains? %2 %1) coord
+               :version
+               (pods/load-pod-metadata pod-spec
+                                       (merge {:cache true}
+                                              (select-keys coord [:version :cache])))
+
+               :path
+               (pods/load-pod-metadata (-> @bb-edn :file io/file)
+                                       pod-spec
+                                       (merge {:cache true}
+                                              (select-keys coord [:path :cache])))
+
+               (throw (IllegalArgumentException.
+                        (str (-> coord keys first)
+                             " is not a supported pod coordinate type. "
+                             "Use :version for registry-hosted pods or :local/root "
+                             "for pods on your local filesystem."))))))
+    {} pods-map))
+
 (def podns (sci/create-ns 'babashka.pods nil))
 
 (def pods-namespace

src/babashka/main.clj

@@ -690,6 +690,8 @@ Use bb run --help to show this help output.
 
 (def env (atom {}))
 
+(def pod-namespaces (volatile! {}))
+
 (defn exec [cli-opts]
   (binding [*unrestricted* true]
     (sci/binding [core/warn-on-reflection @core/warn-on-reflection
@@ -755,6 +757,8 @@ Use bb run --help to show this help output.
                                                               :expressions [(:source res)]})
                                       {})
                                   res))))
+                          (when-let [pod (get @pod-namespaces namespace)]
+                            (pods/load-pod (:pod-spec pod) (:opts pod)))
                           (case namespace
                             clojure.spec.alpha
                             (binding [*out* *err*]
@@ -787,6 +791,9 @@ Use bb run --help to show this help output.
             opts (addons/future opts)
             sci-ctx (sci/init opts)
             _ (vreset! common/ctx sci-ctx)
+            _ (when-let [pods (:pods @common/bb-edn)]
+                (let [pod-metadata (pods/load-pods-metadata pods)]
+                  (vreset! pod-namespaces pod-metadata)))
             preloads (some-> (System/getenv "BABASHKA_PRELOADS") (str/trim))
             [expressions exit-code]
             (cond expressions [expressions nil]

test-resources/pod

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+dir=$(dirname "$0")
+
+export BABASHKA_POD=true
+
+if [[ $BABASHKA_TEST_ENV == "native" ]]; then
+  exec "${dir}"/../bb "${dir}"/pod.clj
+else
+  exec clojure -M:test-pod
+fi

test-resources/pod.clj

@@ -125,7 +125,8 @@
               :shutdown (System/exit 0))))))))
 
 (let [cli-args (set *command-line-args*)]
-  (if (contains? cli-args "--run-as-pod")
+  (if (or (= "true" (System/getenv "BABASHKA_POD"))
+          (contains? cli-args "--run-as-pod"))
     (do (debug "running pod with cli args" cli-args)
         (run-pod cli-args))
     (let [native? (contains? cli-args "--native")

test-resources/pod_tests/bootleg.clj

@@ -0,0 +1,7 @@
+(ns pod-tests.bootleg
+  (:require [pod.retrogradeorbit.bootleg.utils :as utils]))
+
+(defn -main [& args]
+  (-> [:div
+       [:p "Test"]]
+      (utils/convert-to :html)))

test-resources/pod_tests/local.clj

@@ -0,0 +1,5 @@
+(ns pod-tests.local
+  (:require [pod.test-pod :as pod]))
+
+(defn -main [& args]
+  (println (pod/add-sync 40 2)))

test/babashka/bb_edn_test.clj

@@ -425,3 +425,17 @@ even more stuff here\"
                       c {:depends [a b]
                          :task [a b]}}})
     (is (= ["[a 1]" "[b 2]" "[c [1 2]]"] (str/split-lines (test-utils/bb nil "run" "c"))))))
+
+(deftest pod-from-registry-test
+  (when (= "amd64" (System/getProperty "os.arch")) ; TODO: Build bootleg for aarch64 too or use a different pod
+    (test-utils/with-config
+      (pr-str '{:paths ["test-resources"]
+                :pods  {retrogradeorbit/bootleg {:version "0.1.9"}}})
+      (is (= "\"<div><p>Test</p></div>\"\n"
+             (test-utils/bb nil "-m" "pod-tests.bootleg"))))))
+
+(deftest ^:skip-windows local-pod-test
+  (test-utils/with-config
+    (pr-str '{:paths ["test-resources"]
+              :pods {pod/test-pod {:path "test-resources/pod"}}})
+    (is (= "42\n" (test-utils/bb nil "-m" "pod-tests.local")))))

test/babashka/test_utils.clj

@@ -81,8 +81,8 @@
               (do
                 (println (str os))
                 (throw (ex-info (str es)
-                                  {:stdout (str os)
-                                   :stderr (str es)}))))))
+                                {:stdout (str os)
+                                 :stderr (str es)}))))))
       (finally
         (when (string? input-or-opts) (vars/bindRoot sci/in *in*))
         (vars/bindRoot sci/out *out*)