From 189548ca508187d04c02e3f3fe3109f00bacec82 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Fri, 7 Oct 2022 10:24:34 -0600 Subject: [PATCH] Add support for installing pods from local manifests ...as a part of supporting library-bootstrapped pods --- src/babashka/pods/impl/resolver.clj | 82 +++++++++++++++++------------ src/babashka/pods/sci.clj | 56 ++++++++++++++++---- 2 files changed, 93 insertions(+), 45 deletions(-) diff --git a/src/babashka/pods/impl/resolver.clj b/src/babashka/pods/impl/resolver.clj index 61a170e..6271d9d 100644 --- a/src/babashka/pods/impl/resolver.clj +++ b/src/babashka/pods/impl/resolver.clj @@ -131,9 +131,13 @@ "https://raw.githubusercontent.com/babashka/pod-registry/master/manifests/%s/%s/manifest.edn" qsym version)) +(defn pod-manifest-file + [qsym version] + (io/file @pods-repo-dir (str qsym) (str version) "manifest.edn")) + (defn pod-manifest [qsym version force?] - (let [f (io/file @pods-repo-dir (str qsym) (str version) "manifest.edn")] + (let [f (pod-manifest-file qsym version)] (if (and (not force?) (.exists f)) (edn/read-string (slurp f)) @@ -182,41 +186,51 @@ (.digest digest)) (String. "UTF-8")))) +(defn install-pod-artifacts + [artifacts {cdir :cache-dir, ddir :data-dir, :keys [:force? :pod/options]}] + (let [execs (mapv (fn [artifact] + (let [url (:artifact/url artifact) + file-name (last (str/split url #"/")) + cache-file (io/file cdir file-name) + exe-file-name (:artifact/executable artifact) + executable (io/file ddir exe-file-name)] + (when (or force? (not (.exists executable))) + (warn (format "Downloading pod %s" url)) + (download url cache-file false) + (when-let [expected-sha (:artifact/hash artifact)] + (let [sha (sha256 cache-file)] + (when-not (= (str/replace expected-sha #"^sha256:" "") + sha) + (throw (ex-info (str "Wrong SHA-256 for file" (str cache-file)) + {:sha sha + :expected-sha expected-sha}))))) + (let [filename (.getName cache-file)] + (cond (str/ends-with? filename ".zip") + (unzip {:zip-file cache-file + :destination-dir ddir + :verbose false}) + (or (str/ends-with? filename ".tgz") + (str/ends-with? filename ".tar.gz")) + (un-tgz cache-file ddir + false)) + (.delete cache-file)) + (make-executable ddir [exe-file-name] false) + (warn (format "Successfully installed pod %s" exe-file-name)) + (io/file ddir exe-file-name)) + (io/file ddir exe-file-name))) + artifacts)] + {:executable (.getAbsolutePath ^java.io.File (first execs)) + :options options})) + (defn resolve [qsym version force?] (when-not (string? version) (throw (IllegalArgumentException. "Version must be provided for resolving from pod registry!"))) (when-let [manifest (pod-manifest qsym version force?)] - (let [artifacts (match-artifacts manifest) - cdir (cache-dir manifest) + (let [cdir (cache-dir manifest) ddir (data-dir manifest) - execs (mapv (fn [artifact] - (let [url (:artifact/url artifact) - file-name (last (str/split url #"/")) - cache-file (io/file cdir file-name) - executable (io/file ddir (:artifact/executable artifact))] - (when (or force? (not (.exists executable))) - (warn (format "Downloading pod %s (%s)" qsym version)) - (download url cache-file false) - (when-let [expected-sha (:artifact/hash artifact)] - (let [sha (sha256 cache-file)] - (when-not (= (str/replace expected-sha #"^sha256:" "") - sha) - (throw (ex-info (str "Wrong SHA-256 for file" (str cache-file)) - {:sha sha - :expected-sha expected-sha}))))) - (let [filename (.getName cache-file)] - (cond (str/ends-with? filename ".zip") - (unzip {:zip-file cache-file - :destination-dir ddir - :verbose false}) - (or (str/ends-with? filename ".tgz") - (str/ends-with? filename ".tar.gz")) - (un-tgz cache-file ddir - false)) - (.delete cache-file)) - (make-executable ddir [(:artifact/executable artifact)] false) - (warn (format "Successfully installed pod %s (%s)" qsym version)) - (io/file ddir (:artifact/executable artifact))) - (io/file ddir (:artifact/executable artifact)))) artifacts)] - {:executable (.getAbsolutePath ^java.io.File (first execs)) - :options (:pod/options manifest)}))) + artifacts (match-artifacts manifest) + pod-options (:pod/options manifest)] + (install-pod-artifacts artifacts {:cache-dir cdir + :data-dir ddir + :force? force? + :pod/options pod-options})))) diff --git a/src/babashka/pods/sci.clj b/src/babashka/pods/sci.clj index 986d2d4..ac2ce2e 100644 --- a/src/babashka/pods/sci.clj +++ b/src/babashka/pods/sci.clj @@ -41,17 +41,32 @@ (with-open [r (PushbackInputStream. (io/input-stream cache-file))] (impl/read r))))) -(defn load-pod-metadata* [bb-edn-file pod-spec {:keys [:version :cache] :as opts}] - (let [metadata (impl/load-pod-metadata pod-spec opts) - cache-file (when (and metadata cache) - (metadata-cache-file bb-edn-file pod-spec opts))] +(defn cache-pod-metadata! [metadata {:keys [:bb-edn-file :pod-name :pod-version]}] + (let [cache-file (metadata-cache-file bb-edn-file pod-name + {:version pod-version})] (when cache-file (io/make-parents cache-file) (when (fs/writable? (fs/parent cache-file)) (with-open [w (io/output-stream cache-file)] - (impl/write w metadata)))) + (impl/write w metadata)))))) + +(defn load-pod-metadata* [bb-edn-file pod-spec {:keys [:version :cache] :as opts}] + (let [metadata (impl/load-pod-metadata pod-spec opts)] + (when (and metadata cache) + (cache-pod-metadata! metadata {:bb-edn-file bb-edn-file + :pod-name pod-spec + :pod-version version})) metadata)) +(defn pod-namespaces + [pod-name metadata opts] + (reduce + (fn [pod-namespaces ns] + (let [ns-sym (-> ns (get "name") impl/bytes->string symbol)] + (assoc pod-namespaces ns-sym {:pod-spec pod-name + :opts (assoc opts :metadata metadata)}))) + {} (get metadata "namespaces"))) + (defn load-pod-metadata ([pod-spec opts] (load-pod-metadata nil pod-spec opts)) ([bb-edn-file pod-spec {:keys [:cache] :as opts}] @@ -62,12 +77,7 @@ opts))] cached-metadata (load-pod-metadata* bb-edn-file pod-spec opts))] - (reduce - (fn [pod-namespaces ns] - (let [ns-sym (-> ns (get "name") impl/bytes->string symbol)] - (assoc pod-namespaces ns-sym {:pod-spec pod-spec - :opts (assoc opts :metadata metadata)}))) - {} (get metadata "namespaces"))))) + (pod-namespaces pod-spec metadata opts)))) (defn load-pod ([ctx pod-spec] (load-pod ctx pod-spec nil)) @@ -122,6 +132,30 @@ (sci/future (impl/processor pod)) {:pod/id (:pod-id pod)}))) +(defn load-pod-from-manifest + [manifest {:keys [:bb-edn-file]}] + (let [artifacts (resolver/match-artifacts manifest) + pod-name (:pod/name manifest)] + (when artifacts + (let [cdir (resolver/cache-dir manifest) + ddir (resolver/data-dir manifest) + pod (resolver/install-pod-artifacts + artifacts {:cache-dir cdir + :data-dir ddir + :pod/options (:pod/options manifest)}) + metadata (impl/run-pod-for-metadata [(:executable pod)] nil)] + (let [pod-version (:pod/version manifest) + opts {:bb-edn-file bb-edn-file + :pod-name pod-name + :pod-version pod-version}] + (cache-pod-metadata! metadata opts) + ;; TODO: Support local library pods too w/ a :path key here + (pod-namespaces pod-name metadata {:version pod-version})))))) + +(defn pod-manifest-file + [manifest] + (resolver/pod-manifest-file (-> manifest :pod/name symbol) (:pod/version manifest))) + (defn unload-pod ([pod-id] (unload-pod pod-id {})) ([pod-id _opts]