diff --git a/project.clj b/project.clj index 2671cb0..0a225c0 100644 --- a/project.clj +++ b/project.clj @@ -44,7 +44,7 @@ "test-auto" ["with-profile" "+test" "autoexpect"] "start-dev" ["with-profile" "+dev" "repl" ":headless"] "codox" ["with-profile" "+test" "doc"] - "deploy-lib" ["with-profile" "+dev,+build" "deploy" "clojars"]} + "deploy-lib" ["with-profile" "+dev,+build" "do" "deploy" "clojars," "install"]} :repositories {"sonatype" diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 0bcdceb..eb0059c 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -3,6 +3,7 @@ from Deep-Freeze." {:author "Peter Taoussanis"} (:require [clojure.tools.reader.edn :as edn] + [taoensso.encore :as encore] [taoensso.nippy (utils :as utils) (compression :as compression :refer (snappy-compressor)) @@ -302,8 +303,8 @@ (defn- wrap-header [data-ba metadata] (if-let [meta-id (head-meta-id (assoc metadata :version head-version))] - (let [head-ba (utils/ba-concat head-sig (byte-array [meta-id]))] - (utils/ba-concat head-ba data-ba)) + (let [head-ba (encore/ba-concat head-sig (byte-array [meta-id]))] + (encore/ba-concat head-ba data-ba)) (throw (Exception. (str "Unrecognized header metadata: " metadata))))) (comment (wrap-header (.getBytes "foo") {:compressed? true @@ -356,10 +357,10 @@ `(long (BigInteger. (read-bytes ~in :small)))) (defmacro ^:private read-coll [in coll] - `(let [in# ~in] (utils/repeatedly-into ~coll (.readInt in#) (thaw-from-in in#)))) + `(let [in# ~in] (encore/repeatedly-into* ~coll (.readInt in#) (thaw-from-in in#)))) (defmacro ^:private read-kvs [in coll] - `(let [in# ~in] (utils/repeatedly-into ~coll (/ (.readInt in#) 2) + `(let [in# ~in] (encore/repeatedly-into* ~coll (/ (.readInt in#) 2) [(thaw-from-in in#) (thaw-from-in in#)]))) (declare ^:private custom-readers) @@ -371,7 +372,7 @@ (when-debug-mode (println (format "DEBUG - thawing type-id: %s" type-id))) - (utils/case-eval type-id + (encore/case-eval type-id id-reader (let [edn (read-utf8 in)] @@ -446,7 +447,7 @@ ;;; DEPRECATED id-old-reader (edn/read-string (.readUTF in)) id-old-string (.readUTF in) - id-old-map (apply hash-map (utils/repeatedly-into [] + id-old-map (apply hash-map (encore/repeatedly-into* [] (* 2 (.readInt in)) (thaw-from-in in))) id-old-keyword (keyword (.readUTF in)) @@ -472,9 +473,9 @@ (thaw-from-in data-input)) (defn- try-parse-header [ba] - (when-let [[head-ba data-ba] (utils/ba-split ba 4)] - (let [[head-sig* [meta-id]] (utils/ba-split head-ba 3)] - (when (utils/ba= head-sig* head-sig) ; Appears to be well-formed + (when-let [[head-ba data-ba] (encore/ba-split ba 4)] + (let [[head-sig* [meta-id]] (encore/ba-split head-ba 3)] + (when (encore/ba= head-sig* head-sig) ; Appears to be well-formed [data-ba (head-meta meta-id {:unrecognized-meta? true})])))) (defn thaw @@ -699,16 +700,16 @@ ;;;; Tools -(utils/defalias freezeable? utils/freezable?) +(encore/defalias freezeable? utils/freezable?) (defn inspect-ba "Alpha - subject to change." [ba & [thaw-opts]] - (if-not (utils/bytes? ba) :not-ba - (let [[first2bytes nextbytes] (utils/ba-split ba 2) + (if-not (encore/bytes? ba) :not-ba + (let [[first2bytes nextbytes] (encore/ba-split ba 2) known-wrapper (cond - (utils/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin - (utils/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj) + (encore/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin + (encore/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj) unwrapped-ba (if known-wrapper nextbytes ba) [data-ba nippy-header] (or (try-parse-header unwrapped-ba) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index f21750f..03daf53 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -2,9 +2,9 @@ {:author "Peter Taoussanis"} (:require [clojure.tools.reader.edn :as edn] [clojure.data.fressian :as fressian] + [taoensso.encore :as encore] [taoensso.nippy :as nippy :refer (freeze thaw)] - [taoensso.nippy.compression :as compression] - [taoensso.nippy.utils :as utils])) + [taoensso.nippy.compression :as compression])) (def data nippy/stress-data-benchable) @@ -21,7 +21,7 @@ (comment (fressian-thaw (fressian-freeze data))) -(defmacro bench* [& body] `(utils/bench 10000 {:warmup-laps 20000} ~@body)) +(defmacro bench* [& body] `(encore/bench 10000 {:warmup-laps 20000} ~@body)) (defn bench1 [freezer thawer & [sizer]] (let [data-frozen (freezer data) time-freeze (bench* (freezer data)) diff --git a/src/taoensso/nippy/compression.clj b/src/taoensso/nippy/compression.clj index 0a88531..3cc5f87 100644 --- a/src/taoensso/nippy/compression.clj +++ b/src/taoensso/nippy/compression.clj @@ -1,7 +1,6 @@ (ns taoensso.nippy.compression "Alpha - subject to change." {:author "Peter Taoussanis"} - (:require [taoensso.nippy.utils :as utils]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream DataOutputStream])) diff --git a/src/taoensso/nippy/encryption.clj b/src/taoensso/nippy/encryption.clj index 0a056c4..5516783 100644 --- a/src/taoensso/nippy/encryption.clj +++ b/src/taoensso/nippy/encryption.clj @@ -3,7 +3,7 @@ Simple no-nonsense crypto with reasonable defaults. Because your Clojure data deserves some privacy." {:author "Peter Taoussanis"} - (:require [taoensso.nippy.utils :as utils])) + (:require [taoensso.encore :as encore])) ;;;; Interface @@ -32,7 +32,7 @@ (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds." [salt-ba ^String pwd] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] - (if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba)) + (if salt-ba (encore/ba-concat salt-ba pwd-ba) pwd-ba)) n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] (if-not (zero? n) (recur (.digest sha512-md ba) (dec n)) @@ -70,22 +70,22 @@ salt? (= type :salted) iv-ba (rand-bytes aes128-block-size) salt-ba (when salt? (rand-bytes salt-size)) - prefix-ba (if-not salt? iv-ba (utils/ba-concat iv-ba salt-ba)) - key (utils/memoized (when-not salt? (:key-cache this)) + prefix-ba (if-not salt? iv-ba (encore/ba-concat iv-ba salt-ba)) + key (encore/memoized (when-not salt? (:key-cache this)) sha512-key salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE ^javax.crypto.spec.SecretKeySpec key iv) - (utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba)))) + (encore/ba-concat prefix-ba (.doFinal aes128-cipher data-ba)))) (decrypt [this typed-pwd ba] (let [[type pwd] (destructure-typed-pwd typed-pwd) salt? (= type :salted) prefix-size (+ aes128-block-size (if salt? salt-size 0)) - [prefix-ba data-ba] (utils/ba-split ba prefix-size) + [prefix-ba data-ba] (encore/ba-split ba prefix-size) [iv-ba salt-ba] (if-not salt? [prefix-ba nil] - (utils/ba-split prefix-ba aes128-block-size)) - key (utils/memoized (when-not salt? (:key-cache this)) + (encore/ba-split prefix-ba aes128-block-size)) + key (encore/memoized (when-not salt? (:key-cache this)) sha512-key salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE diff --git a/src/taoensso/nippy/tools.clj b/src/taoensso/nippy/tools.clj index 58d1456..a6f7516 100644 --- a/src/taoensso/nippy/tools.clj +++ b/src/taoensso/nippy/tools.clj @@ -3,8 +3,7 @@ Utilities for third-party tools that want to add fully-user-configurable Nippy support. Used by Carmine and Faraday." {:author "Peter Taoussanis"} - (:require [taoensso.nippy :as nippy] - [taoensso.nippy.utils :as utils])) + (:require [taoensso.nippy :as nippy])) (defrecord WrappedForFreezing [value opts]) (defn wrapped-for-freezing? [x] (instance? WrappedForFreezing x)) diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 555e83b..4ed2138 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -1,114 +1,10 @@ (ns taoensso.nippy.utils {:author "Peter Taoussanis"} - (:require [clojure.string :as str] + (:require [clojure.string :as str] [clojure.tools.reader.edn :as edn]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable ObjectOutputStream ObjectInputStream])) -(defmacro defalias - "Defines an alias for a var, preserving metadata. Adapted from - clojure.contrib/def.clj, Ref. http://goo.gl/xpjeH" - [name target & [doc]] - `(let [^clojure.lang.Var v# (var ~target)] - (alter-meta! (def ~name (.getRawRoot v#)) - #(merge % (apply dissoc (meta v#) [:column :line :file :test :name]) - (when-let [doc# ~doc] {:doc doc#}))) - (var ~name))) - -(defmacro case-eval - "Like `case` but evaluates test constants for their compile-time value." - [e & clauses] - (let [;; Don't evaluate default expression! - default (when (odd? (count clauses)) (last clauses)) - clauses (if default (butlast clauses) clauses)] - `(case ~e - ~@(map-indexed (fn [i# form#] (if (even? i#) (eval form#) form#)) - clauses) - ~(when default default)))) - -(defmacro repeatedly-into - "Like `repeatedly` but faster and `conj`s items into given collection." - [coll n & body] - `(let [coll# ~coll - n# ~n] - (if (instance? clojure.lang.IEditableCollection coll#) - (loop [v# (transient coll#) idx# 0] - (if (>= idx# n#) - (persistent! v#) - (recur (conj! v# ~@body) - (inc idx#)))) - (loop [v# coll# - idx# 0] - (if (>= idx# n#) - v# - (recur (conj v# ~@body) - (inc idx#))))))) - -(defmacro time-ns "Returns number of nanoseconds it takes to execute body." - [& body] `(let [t0# (System/nanoTime)] ~@body (- (System/nanoTime) t0#))) - -(defmacro bench - "Repeatedly executes body and returns time taken to complete execution." - [nlaps {:keys [nlaps-warmup nthreads as-ns?] - :or {nlaps-warmup 0 - nthreads 1}} & body] - `(let [nlaps# ~nlaps - nlaps-warmup# ~nlaps-warmup - nthreads# ~nthreads] - (try (dotimes [_# nlaps-warmup#] ~@body) - (let [nanosecs# - (if (= nthreads# 1) - (time-ns (dotimes [_# nlaps#] ~@body)) - (let [nlaps-per-thread# (int (/ nlaps# nthreads#))] - (time-ns - (->> (fn [] (future (dotimes [_# nlaps-per-thread#] ~@body))) - (repeatedly nthreads#) - (doall) - (map deref) - (dorun)))))] - (if ~as-ns? nanosecs# (Math/round (/ nanosecs# 1000000.0)))) - (catch Exception e# (format "DNF: %s" (.getMessage e#)))))) - -(defn memoized - "Like `(partial memoize* {})` but takes an explicit cache atom (possibly nil) - and immediately applies memoized f to given arguments." - [cache f & args] - (if-not cache - (apply f args) - (if-let [dv (@cache args)] - @dv - (locking cache ; For thread racing - (if-let [dv (@cache args)] ; Retry after lock acquisition! - @dv - (let [dv (delay (apply f args))] - (swap! cache assoc args dv) - @dv)))))) - -(comment (memoized nil +) - (memoized nil + 5 12)) - -(def ^:const bytes-class (Class/forName "[B")) -(defn bytes? [x] (instance? bytes-class x)) -(defn ba= [^bytes x ^bytes y] (java.util.Arrays/equals x y)) - -(defn ba-concat ^bytes [^bytes ba1 ^bytes ba2] - (let [s1 (alength ba1) - s2 (alength ba2) - out (byte-array (+ s1 s2))] - (System/arraycopy ba1 0 out 0 s1) - (System/arraycopy ba2 0 out s1 s2) - out)) - -(defn ba-split [^bytes ba ^Integer idx] - (let [s (alength ba)] - (when (> s idx) - [(java.util.Arrays/copyOf ba idx) - (java.util.Arrays/copyOfRange ba idx s)]))) - -(comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar"))) - (let [[x y] (ba-split (.getBytes "foobar") 5)] - [(String. x) (String. y)])) - ;;;; Fallback type tests ;; Unfortunately the only reliable way we can tell if something's ;; really serializable/readable is to actually try a full roundtrip. diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 01f823a..ae79828 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -7,6 +7,8 @@ [taoensso.nippy.compression :as compression] [taoensso.nippy.benchmarks :as benchmarks])) +(comment (test/run-tests '[taoensso.nippy.tests.main])) + (def test-data nippy/stress-data-comparable) (defn- before-run {:expectations-options :before-run} []) (defn- after-run {:expectations-options :after-run} [])