luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refer rename: encore->enc

Browse source
This commit is contained in:
Peter Taoussanis 2015-09-29 14:30:25 +07:00
parent 734e88b20c
commit 50ffb78c22
5 changed files with 61 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
src/taoensso/nippy.clj
View file

@ -2,7 +2,7 @@
"High-performance JVM Clojure serialization library. Originally adapted from "High-performance JVM Clojure serialization library. Originally adapted from
Deep-Freeze (https://goo.gl/OePPGr)." Deep-Freeze (https://goo.gl/OePPGr)."
{:author "Peter Taoussanis (@ptaoussanis)"} {:author "Peter Taoussanis (@ptaoussanis)"}
(:require [taoensso.encore :as encore] (:require [taoensso.encore :as enc]
[taoensso.nippy [taoensso.nippy
(utils :as utils) (utils :as utils)
(compression :as compression) (compression :as compression)
@ -18,9 +18,9 @@
PersistentQueue PersistentTreeMap PersistentTreeSet PersistentList ; LazySeq PersistentQueue PersistentTreeMap PersistentTreeSet PersistentList ; LazySeq
IRecord ISeq])) IRecord ISeq]))
(if (vector? taoensso.encore/encore-version) (if (vector? enc/encore-version)
(encore/assert-min-encore-version [2 16 0]) (enc/assert-min-encore-version [2 16 0])
(encore/assert-min-encore-version 2.16)) (enc/assert-min-encore-version 2.16))
;;;; Nippy data format ;;;; Nippy data format
;; * 4-byte header (Nippy v2.x+) (may be disabled but incl. by default) [1] ;; * 4-byte header (Nippy v2.x+) (may be disabled but incl. by default) [1]
@ -137,18 +137,18 @@
;;;; Ns imports (mostly for convenience of lib consumers) ;;;; Ns imports (mostly for convenience of lib consumers)
(encore/defalias compress compression/compress) (enc/defalias compress compression/compress)
(encore/defalias decompress compression/decompress) (enc/defalias decompress compression/decompress)
(encore/defalias snappy-compressor compression/snappy-compressor) (enc/defalias snappy-compressor compression/snappy-compressor)
(encore/defalias lzma2-compressor compression/lzma2-compressor) (enc/defalias lzma2-compressor compression/lzma2-compressor)
(encore/defalias lz4-compressor compression/lz4-compressor) (enc/defalias lz4-compressor compression/lz4-compressor)
(encore/defalias lz4hc-compressor compression/lz4hc-compressor) (enc/defalias lz4hc-compressor compression/lz4hc-compressor)
(encore/defalias encrypt encryption/encrypt) (enc/defalias encrypt encryption/encrypt)
(encore/defalias decrypt encryption/decrypt) (enc/defalias decrypt encryption/decrypt)
(encore/defalias aes128-encryptor encryption/aes128-encryptor) (enc/defalias aes128-encryptor encryption/aes128-encryptor)
(encore/defalias freezable? utils/freezable?) (enc/defalias freezable? utils/freezable?)
;;;; Freezing ;;;; Freezing
@ -190,7 +190,7 @@
`(if (counted? ~'x) `(if (counted? ~'x)
(do (do
(. ~'out ~wc (count ~'x)) (. ~'out ~wc (count ~'x))
(encore/run!* (fn [i#] (freeze-to-out! ~'out i#)) ~'x)) (enc/run!* (fn [i#] (freeze-to-out! ~'out i#)) ~'x))
(let [bas# (ByteArrayOutputStream. 64) (let [bas# (ByteArrayOutputStream. 64)
sout# (DataOutputStream. bas#) sout# (DataOutputStream. bas#)
cnt# (reduce (fn [^long cnt# i#] cnt# (reduce (fn [^long cnt# i#]
@ -205,7 +205,7 @@
(let [wc (if small? 'writeByte 'writeInt)] (let [wc (if small? 'writeByte 'writeInt)]
`(do `(do
(. ~'out ~wc (count ~'x)) (. ~'out ~wc (count ~'x))
(encore/run-kv! (enc/run-kv!
(fn [k# v#] (fn [k# v#]
(freeze-to-out! ~'out k#) (freeze-to-out! ~'out k#)
(freeze-to-out! ~'out v#)) (freeze-to-out! ~'out v#))
@ -335,9 +335,9 @@
(.writeLong out (.getMostSignificantBits x)) (.writeLong out (.getMostSignificantBits x))
(.writeLong out (.getLeastSignificantBits x))) (.writeLong out (.getLeastSignificantBits x)))
(encore/defonce* ^:dynamic *final-freeze-fallback* nil) (enc/defonce* ^:dynamic *final-freeze-fallback* nil)
(defn freeze-fallback-as-str [out x] (defn freeze-fallback-as-str [out x]
(-freeze-to-out {:nippy/unfreezable (encore/pr-edn x) :type (type x)} out)) (-freeze-to-out {:nippy/unfreezable (enc/pr-edn x) :type (type x)} out))
(comment (comment
(require '[clojure.core.async :as async]) (require '[clojure.core.async :as async])
@ -361,25 +361,25 @@
(do (when-debug-mode (do (when-debug-mode
(println (format "DEBUG - Reader fallback: %s" (type x)))) (println (format "DEBUG - Reader fallback: %s" (type x))))
(write-id out id-reader) (write-id out id-reader)
(write-utf8 out (encore/pr-edn x))) (write-utf8 out (enc/pr-edn x)))
:else ; Fallback #3: *final-freeze-fallback* :else ; Fallback #3: *final-freeze-fallback*
(if-let [ffb *final-freeze-fallback*] (if-let [ffb *final-freeze-fallback*]
(ffb x out) (ffb x out)
(throw (ex-info (format "Unfreezable type: %s %s" (type x) (str x)) (throw (ex-info (format "Unfreezable type: %s %s" (type x) (str x))
{:type (type x) {:type (type x)
:as-str (encore/pr-edn x)})))))) :as-str (enc/pr-edn x)}))))))
(def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta)) (def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta))
(def ^:private get-head-ba (def ^:private get-head-ba
(memoize (memoize
(fn [head-meta] (fn [head-meta]
(when-let [meta-id (get head-meta-id (assoc head-meta :version head-version))] (when-let [meta-id (get head-meta-id (assoc head-meta :version head-version))]
(encore/ba-concat head-sig (byte-array [meta-id])))))) (enc/ba-concat head-sig (byte-array [meta-id]))))))
(defn- wrap-header [data-ba head-meta] (defn- wrap-header [data-ba head-meta]
(if-let [head-ba (get-head-ba head-meta)] (if-let [head-ba (get-head-ba head-meta)]
(encore/ba-concat head-ba data-ba) (enc/ba-concat head-ba data-ba)
(throw (ex-info (format "Unrecognized header meta: %s" head-meta) (throw (ex-info (format "Unrecognized header meta: %s" head-meta)
{:head-meta head-meta})))) {:head-meta head-meta}))))
@ -399,7 +399,7 @@
(> ba-len 1024) lz4-compressor (> ba-len 1024) lz4-compressor
:else nil))) :else nil)))
(encore/defonce* ^:dynamic *default-freeze-compressor-selector* (enc/defonce* ^:dynamic *default-freeze-compressor-selector*
"(fn selector [^bytes ba])->compressor used by `(freeze <x> {:compressor :auto})." "(fn selector [^bytes ba])->compressor used by `(freeze <x> {:compressor :auto})."
default-freeze-compressor-selector) default-freeze-compressor-selector)
@ -475,23 +475,23 @@
(defmacro ^:private read-coll [in coll & [small?]] (defmacro ^:private read-coll [in coll & [small?]]
(let [rc (if small? 'readByte 'readInt)] (let [rc (if small? 'readByte 'readInt)]
`(let [in# ~in] `(let [in# ~in]
(encore/repeatedly-into ~coll (. in# ~rc) (enc/repeatedly-into ~coll (. in# ~rc)
(fn [] (thaw-from-in in#)))))) (fn [] (thaw-from-in in#))))))
(defmacro ^:private read-kvs [in coll & [small?]] (defmacro ^:private read-kvs [in coll & [small?]]
(let [rc (if small? 'readByte 'readInt)] (let [rc (if small? 'readByte 'readInt)]
`(let [in# ~in] `(let [in# ~in]
(encore/repeatedly-into ~coll (. in# ~rc) (enc/repeatedly-into ~coll (. in# ~rc)
(fn [] [(thaw-from-in in#) (thaw-from-in in#)]))))) (fn [] [(thaw-from-in in#) (thaw-from-in in#)])))))
(defmacro ^:private read-kvs-depr1 [in coll] (defmacro ^:private read-kvs-depr1 [in coll]
`(let [in# ~in] `(let [in# ~in]
(encore/repeatedly-into ~coll (quot (.readInt in#) 2) (enc/repeatedly-into ~coll (quot (.readInt in#) 2)
(fn [] [(thaw-from-in in#) (thaw-from-in in#)])))) (fn [] [(thaw-from-in in#) (thaw-from-in in#)]))))
(def ^:private class-method-sig (into-array Class [IPersistentMap])) (def ^:private class-method-sig (into-array Class [IPersistentMap]))
(encore/defonce* ^:dynamic *custom-readers* "{<hash-or-byte-id> (fn [data-input])}" nil) (enc/defonce* ^:dynamic *custom-readers* "{<hash-or-byte-id> (fn [data-input])}" nil)
(defn swap-custom-readers! [f] (alter-var-root #'*custom-readers* f)) (defn swap-custom-readers! [f] (alter-var-root #'*custom-readers* f))
(defn- read-custom! [type-id in] (defn- read-custom! [type-id in]
@ -516,12 +516,12 @@
(when-debug-mode (when-debug-mode
(println (format "DEBUG - thawing type-id: %s" type-id))) (println (format "DEBUG - thawing type-id: %s" type-id)))
(encore/case-eval type-id (enc/case-eval type-id
id-reader id-reader
(let [edn (read-utf8 in)] (let [edn (read-utf8 in)]
(try (try
(encore/read-edn {:readers *data-readers*} edn) (enc/read-edn {:readers *data-readers*} edn)
(catch Exception e (catch Exception e
{:type :reader {:type :reader
:throwable e :throwable e
@ -615,9 +615,9 @@
;;; DEPRECATED ;;; DEPRECATED
id-sorted-map-depr1 (read-kvs-depr1 in (sorted-map)) id-sorted-map-depr1 (read-kvs-depr1 in (sorted-map))
id-map-depr2 (read-kvs-depr1 in {}) id-map-depr2 (read-kvs-depr1 in {})
id-reader-depr1 (encore/read-edn (.readUTF in)) id-reader-depr1 (enc/read-edn (.readUTF in))
id-string-depr1 (.readUTF in) id-string-depr1 (.readUTF in)
id-map-depr1 (apply hash-map (encore/repeatedly-into [] (* 2 (.readInt in)) id-map-depr1 (apply hash-map (enc/repeatedly-into [] (* 2 (.readInt in))
(fn [] (thaw-from-in in)))) (fn [] (thaw-from-in in))))
id-keyword-depr1 (keyword (.readUTF in)) id-keyword-depr1 (keyword (.readUTF in))
@ -639,9 +639,9 @@
(thaw-from-in data-input)) (thaw-from-in data-input))
(defn- try-parse-header [ba] (defn- try-parse-header [ba]
(when-let [[head-ba data-ba] (encore/ba-split ba 4)] (when-let [[head-ba data-ba] (enc/ba-split ba 4)]
(let [[head-sig* [meta-id]] (encore/ba-split head-ba 3)] (let [[head-sig* [meta-id]] (enc/ba-split head-ba 3)]
(when (encore/ba= head-sig* head-sig) ; Header appears to be well-formed (when (enc/ba= head-sig* head-sig) ; Header appears to be well-formed
[data-ba (get head-meta meta-id {:unrecognized-meta? true})])))) [data-ba (get head-meta meta-id {:unrecognized-meta? true})]))))
(defn- get-auto-compressor [compressor-id] (defn- get-auto-compressor [compressor-id]
@ -928,12 +928,12 @@
(defn inspect-ba "Alpha - subject to change" (defn inspect-ba "Alpha - subject to change"
[ba & [thaw-opts]] [ba & [thaw-opts]]
(if-not (encore/bytes? ba) :not-ba (if-not (enc/bytes? ba) :not-ba
(let [[first2bytes nextbytes] (encore/ba-split ba 2) (let [[first2bytes nextbytes] (enc/ba-split ba 2)
known-wrapper known-wrapper
(cond (cond
(encore/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin (enc/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin
(encore/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj) (enc/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj)
unwrapped-ba (if known-wrapper nextbytes ba) unwrapped-ba (if known-wrapper nextbytes ba)
[data-ba nippy-header] (or (try-parse-header unwrapped-ba) [data-ba nippy-header] (or (try-parse-header unwrapped-ba)

10
src/taoensso/nippy/benchmarks.clj
View file

@ -1,7 +1,7 @@
(ns taoensso.nippy.benchmarks (ns taoensso.nippy.benchmarks
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [clojure.data.fressian :as fressian] (:require [clojure.data.fressian :as fressian]
[taoensso.encore :as encore] [taoensso.encore :as enc]
[taoensso.nippy :as nippy :refer (freeze thaw)])) [taoensso.nippy :as nippy :refer (freeze thaw)]))
(def data nippy/stress-data-benchable) (def data nippy/stress-data-benchable)
@ -19,7 +19,7 @@
(comment (fressian-thaw (fressian-freeze data))) (comment (fressian-thaw (fressian-freeze data)))
(defmacro bench* [& body] `(encore/bench 10000 {:warmup-laps 20000} ~@body)) (defmacro bench* [& body] `(enc/bench 10000 {:warmup-laps 20000} ~@body))
(defn bench1 [freezer thawer & [sizer]] (defn bench1 [freezer thawer & [sizer]]
(let [data-frozen (freezer data) (let [data-frozen (freezer data)
time-freeze (bench* (freezer data)) time-freeze (bench* (freezer data))
@ -36,7 +36,7 @@
(println (str "\nLap " (inc l) "/" laps "...")) (println (str "\nLap " (inc l) "/" laps "..."))
(when reader? ; Slow (when reader? ; Slow
(println {:reader (bench1 encore/pr-edn encore/read-edn (println {:reader (bench1 enc/pr-edn enc/read-edn
#(count (.getBytes ^String % "UTF-8")))})) #(count (.getBytes ^String % "UTF-8")))}))
(println {:default (bench1 #(freeze % {}) (println {:default (bench1 #(freeze % {})
@ -58,7 +58,7 @@
(println "\nDone! (Time for cake?)") (println "\nDone! (Time for cake?)")
true) true)
(comment (encore/read-edn (encore/pr-edn data)) (comment (enc/read-edn (enc/pr-edn data))
(bench1 fressian-freeze fressian-thaw)) (bench1 fressian-freeze fressian-thaw))
(comment (comment

8
src/taoensso/nippy/compression.clj
View file

@ -1,6 +1,6 @@
(ns taoensso.nippy.compression (ns taoensso.nippy.compression
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [taoensso.encore :as encore]) (:require [taoensso.encore :as enc])
(:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream (:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
DataOutputStream])) DataOutputStream]))
@ -122,10 +122,10 @@
(comment (comment
(def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8")) (def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8"))
(defn bench1 [compressor] (defn bench1 [compressor]
{:time (encore/bench 10000 {:nlaps-warmup 10000} {:time (enc/bench 10000 {:nlaps-warmup 10000}
(->> ba-bench (compress compressor) (decompress compressor))) (->> ba-bench (compress compressor) (decompress compressor)))
:ratio (encore/round2 (/ (count (compress compressor ba-bench)) :ratio (enc/round2 (/ (count (compress compressor ba-bench))
(count ba-bench)))}) (count ba-bench)))})
(println (println
{:snappy (bench1 snappy-compressor) {:snappy (bench1 snappy-compressor)

22
src/taoensso/nippy/encryption.clj
View file

@ -2,7 +2,7 @@
"Simple no-nonsense crypto with reasonable defaults. Because your Clojure data "Simple no-nonsense crypto with reasonable defaults. Because your Clojure data
deserves some privacy." deserves some privacy."
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [taoensso.encore :as encore])) (:require [taoensso.encore :as enc]))
;;;; Interface ;;;; Interface
@ -16,15 +16,15 @@
;;;; Default digests, ciphers, etc. ;;;; Default digests, ciphers, etc.
(def ^:private aes128-cipher* (def ^:private aes128-cipher*
(encore/thread-local-proxy (enc/thread-local-proxy
(javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))) (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding")))
(def ^:private sha512-md* (def ^:private sha512-md*
(encore/thread-local-proxy (enc/thread-local-proxy
(java.security.MessageDigest/getInstance "SHA-512"))) (java.security.MessageDigest/getInstance "SHA-512")))
(def ^:private prng* (def ^:private prng*
(encore/thread-local-proxy (enc/thread-local-proxy
(java.security.SecureRandom/getInstance "SHA1PRNG"))) (java.security.SecureRandom/getInstance "SHA1PRNG")))
(defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*)) (defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*))
@ -44,7 +44,7 @@
[salt-ba ^String pwd] [salt-ba ^String pwd]
(let [md (sha512-md)] (let [md (sha512-md)]
(loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")]
(if salt-ba (encore/ba-concat salt-ba pwd-ba) pwd-ba)) (if salt-ba (enc/ba-concat salt-ba pwd-ba) pwd-ba))
n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] n (* (int Short/MAX_VALUE) (if salt-ba 5 64))]
(if-not (zero? n) (if-not (zero? n)
(recur (.digest md ba) (dec n)) (recur (.digest md ba) (dec n))
@ -82,26 +82,26 @@
salt? (identical? type :salted) salt? (identical? type :salted)
iv-ba (rand-bytes aes128-block-size) iv-ba (rand-bytes aes128-block-size)
salt-ba (when salt? (rand-bytes salt-size)) salt-ba (when salt? (rand-bytes salt-size))
prefix-ba (if-not salt? iv-ba (encore/ba-concat iv-ba salt-ba)) prefix-ba (if-not salt? iv-ba (enc/ba-concat iv-ba salt-ba))
key (if salt? key (if salt?
(key-gen salt-ba pwd) (key-gen salt-ba pwd)
(encore/memoized key-cache key-gen salt-ba pwd)) (enc/memoized key-cache key-gen salt-ba pwd))
iv (javax.crypto.spec.IvParameterSpec. iv-ba) iv (javax.crypto.spec.IvParameterSpec. iv-ba)
cipher (aes128-cipher)] cipher (aes128-cipher)]
(.init cipher javax.crypto.Cipher/ENCRYPT_MODE (.init cipher javax.crypto.Cipher/ENCRYPT_MODE
^javax.crypto.spec.SecretKeySpec key iv) ^javax.crypto.spec.SecretKeySpec key iv)
(encore/ba-concat prefix-ba (.doFinal cipher data-ba)))) (enc/ba-concat prefix-ba (.doFinal cipher data-ba))))
(decrypt [_ typed-pwd ba] (decrypt [_ typed-pwd ba]
(let [[type pwd] (destructure-typed-pwd typed-pwd) (let [[type pwd] (destructure-typed-pwd typed-pwd)
salt? (= type :salted) salt? (= type :salted)
prefix-size (+ aes128-block-size (if salt? salt-size 0)) prefix-size (+ aes128-block-size (if salt? salt-size 0))
[prefix-ba data-ba] (encore/ba-split ba prefix-size) [prefix-ba data-ba] (enc/ba-split ba prefix-size)
[iv-ba salt-ba] (if-not salt? [prefix-ba nil] [iv-ba salt-ba] (if-not salt? [prefix-ba nil]
(encore/ba-split prefix-ba aes128-block-size)) (enc/ba-split prefix-ba aes128-block-size))
key (if salt? key (if salt?
(key-gen salt-ba pwd) (key-gen salt-ba pwd)
(encore/memoized key-cache key-gen salt-ba pwd)) (enc/memoized key-cache key-gen salt-ba pwd))
iv (javax.crypto.spec.IvParameterSpec. iv-ba) iv (javax.crypto.spec.IvParameterSpec. iv-ba)
cipher (aes128-cipher)] cipher (aes128-cipher)]
(.init cipher javax.crypto.Cipher/DECRYPT_MODE (.init cipher javax.crypto.Cipher/DECRYPT_MODE

6
src/taoensso/nippy/utils.clj
View file

@ -1,7 +1,7 @@
(ns taoensso.nippy.utils (ns taoensso.nippy.utils
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [clojure.string :as str] (:require [clojure.string :as str]
[taoensso.encore :as encore]) [taoensso.encore :as enc])
(:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable (:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable
ObjectOutputStream ObjectInputStream])) ObjectOutputStream ObjectInputStream]))
@ -17,7 +17,7 @@
cacheable? (not (re-find #"__\d+" (str t))) ; gensym form cacheable? (not (re-find #"__\d+" (str t))) ; gensym form
test (fn [] (try (f-test x) (catch Exception _ false)))] test (fn [] (try (f-test x) (catch Exception _ false)))]
(if-not cacheable? (test) (if-not cacheable? (test)
@(encore/swap-val! cache t #(if % % (delay (test))))))))) @(enc/swap-val! cache t #(if % % (delay (test)))))))))
(def serializable? (def serializable?
(memoize-type-test (memoize-type-test
@ -33,7 +33,7 @@
(cast class object) (cast class object)
true))))) true)))))
(def readable? (memoize-type-test (fn [x] (-> x encore/pr-edn encore/read-edn) true))) (def readable? (memoize-type-test (fn [x] (-> x enc/pr-edn enc/read-edn) true)))
(comment (comment
(serializable? "Hello world") (serializable? "Hello world")