From 8989df5c3dace07a363121358c5cfb5850a12581 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 18:50:10 +0700 Subject: [PATCH 01/33] v2.11.0-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8443df2..bb35c85 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.10.0" +(defproject com.taoensso/nippy "2.11.0-SNAPSHOT" :author "Peter Taoussanis " :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" From 885f192f6b04dd4b1e0a80b7052c098fd3173977 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 15:58:08 +0700 Subject: [PATCH 02/33] Micro optimization: drop unnecessary double kvs count Had a vestigial count doubling from an historical implementation that constructed hash-maps using `(apply hash-map ...)` --- src/taoensso/nippy.clj | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index b239400..4cb8f35 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -86,9 +86,9 @@ (def ^:const id-seq (int 24)) (def ^:const id-meta (int 25)) (def ^:const id-queue (int 26)) - (def ^:const id-map (int 27)) + (def ^:const id-map (int 30)) (def ^:const id-sorted-set (int 28)) - (def ^:const id-sorted-map (int 29)) + (def ^:const id-sorted-map (int 31)) (def ^:const id-byte (int 40)) (def ^:const id-short (int 41)) @@ -124,10 +124,12 @@ ;; (def ^:const id-map-small (int 112)) ; '' ;;; DEPRECATED (old types will be supported only for thawing) - (def ^:const id-reader-depr1 (int 1)) ; v0.9.2+ for +64k support - (def ^:const id-string-depr1 (int 11)) ; v0.9.2+ for +64k support - (def ^:const id-map-depr1 (int 22)) ; v0.9.0+ for more efficient thaw - (def ^:const id-keyword-depr1 (int 12)) ; v2.0.0-alpha5+ for str consistecy + (def ^:const id-reader-depr1 (int 1)) ; v0.9.2+ for +64k support + (def ^:const id-string-depr1 (int 11)) ; v0.9.2+ for +64k support + (def ^:const id-map-depr1 (int 22)) ; v0.9.0+ for more efficient thaw + (def ^:const id-keyword-depr1 (int 12)) ; v2.0.0-alpha5+ for str consistecy + (def ^:const id-map-depr2 (int 27)) ; v2.11+ for count/2 + (def ^:const id-sorted-map-depr1 (int 29)) ; v2.11+ for count/2 ) ;;;; Ns imports (mostly for convenience of lib consumers) @@ -175,7 +177,7 @@ (defmacro write-compact-long "Uses 2->9 bytes." [out x] `(write-bytes ~out (.toByteArray (java.math.BigInteger/valueOf (long ~x))) - :small)) + :small)) (comment (alength (.toByteArray (java.math.BigInteger/valueOf Long/MAX_VALUE)))) @@ -215,7 +217,7 @@ (defmacro ^:private freezer-kvs [type id & body] `(freezer ~type ~id - (.writeInt ~'out (* 2 (count ~'x))) ; *2 here is vestigial + (.writeInt ~'out (count ~'x)) (encore/run-kv! (fn [k# v#] (freeze-to-out ~'out k#) @@ -466,7 +468,12 @@ (defmacro ^:private read-kvs [in coll] `(let [in# ~in] - (encore/repeatedly-into ~coll (quot (.readInt in#) 2) ; /2 here is vestigial + (encore/repeatedly-into ~coll (.readInt in#) + (fn [] [(thaw-from-in in#) (thaw-from-in in#)])))) + +(defmacro ^:private read-kvs-depr1 [in coll] + `(let [in# ~in] + (encore/repeatedly-into ~coll (quot (.readInt in#) 2) (fn [] [(thaw-from-in in#) (thaw-from-in in#)])))) (def ^:private class-method-sig (into-array Class [IPersistentMap])) @@ -590,6 +597,8 @@ id-uuid (UUID. (.readLong in) (.readLong in)) ;;; DEPRECATED + id-sorted-map-depr1 (read-kvs-depr1 in (sorted-map)) + id-map-depr2 (read-kvs-depr1 in {}) id-reader-depr1 (encore/read-edn (.readUTF in)) id-string-depr1 (.readUTF in) id-map-depr1 (apply hash-map (encore/repeatedly-into [] (* 2 (.readInt in)) From b298d690c72c91fb9a148d772bfda6287b67112b Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 16:25:43 +0700 Subject: [PATCH 03/33] Misc hk, reorganize type ids --- src/taoensso/nippy.clj | 166 ++++++++++++++++++++--------------------- 1 file changed, 79 insertions(+), 87 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 4cb8f35..16baa8f 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -23,20 +23,20 @@ (encore/assert-min-encore-version 2.16)) ;;;; Nippy data format -;; * 4-byte header (Nippy v2.x+) (may be disabled but incl. by default) [1]. +;; * 4-byte header (Nippy v2.x+) (may be disabled but incl. by default) [1] ;; { * 1-byte type id. ;; * Arb-length payload. } ... ;; ;; [1] Inclusion of header is strongly recommended. Purpose: -;; * Sanity check (confirm that data appears to be Nippy data). -;; * Nippy version check (=> supports changes to data schema over time). -;; * Supports :auto thaw compressor, encryptor. +;; * Sanity check (confirm that data appears to be Nippy data) +;; * Nippy version check (=> supports changes to data schema over time) +;; * Supports :auto thaw compressor, encryptor ;; * Supports :auto freeze compressor (since this depends on :auto thaw -;; compressor). +;; compressor) ;; (def ^:private ^:const head-version 1) (def ^:private head-sig (.getBytes "NPY" "UTF-8")) -(def ^:private ^:const head-meta "Final byte stores version-dependent metadata." +(def ^:private ^:const head-meta "Final byte stores version-dependent metadata" {(byte 0) {:version 1 :compressor-id nil :encryptor-id nil} (byte 4) {:version 1 :compressor-id nil :encryptor-id :else} (byte 5) {:version 1 :compressor-id :else :encryptor-id nil} @@ -65,71 +65,72 @@ ;; ** Negative ids reserved for user-defined types ** ;; - (def ^:const id-reserved (int 0)) - ;; 1 ; Deprecated - (def ^:const id-bytes (int 2)) - (def ^:const id-nil (int 3)) - (def ^:const id-boolean (int 4)) - (def ^:const id-reader (int 5)) ; Fallback #2 - (def ^:const id-serializable (int 6)) ; Fallback #1 + (def ^:const id-reserved (int 0)) + ;; 1 ; Deprecated + (def ^:const id-bytes (int 2)) + (def ^:const id-nil (int 3)) + (def ^:const id-boolean (int 4)) + (def ^:const id-reader (int 5)) ; Fallback #2 + (def ^:const id-serializable (int 6)) ; Fallback #1 - (def ^:const id-char (int 10)) - ;; 11 ; Deprecated - ;; 12 ; Deprecated - (def ^:const id-string (int 13)) - (def ^:const id-keyword (int 14)) + (def ^:const id-char (int 10)) + ;; 11 ; Deprecated + ;; 12 ; Deprecated + (def ^:const id-string (int 13)) + (def ^:const id-keyword (int 14)) - (def ^:const id-list (int 20)) - (def ^:const id-vector (int 21)) - ;; 22 ; Deprecated - (def ^:const id-set (int 23)) - (def ^:const id-seq (int 24)) - (def ^:const id-meta (int 25)) - (def ^:const id-queue (int 26)) - (def ^:const id-map (int 30)) - (def ^:const id-sorted-set (int 28)) - (def ^:const id-sorted-map (int 31)) + (def ^:const id-list (int 20)) + (def ^:const id-vector (int 21)) + ;; 22 ; Deprecated + (def ^:const id-set (int 23)) + (def ^:const id-seq (int 24)) + (def ^:const id-meta (int 25)) + (def ^:const id-queue (int 26)) + ;; 27 ; Deprecated + (def ^:const id-sorted-set (int 28)) + ;; 29 ; Deprecated + (def ^:const id-map (int 30)) + (def ^:const id-sorted-map (int 31)) - (def ^:const id-byte (int 40)) - (def ^:const id-short (int 41)) - (def ^:const id-integer (int 42)) - (def ^:const id-long (int 43)) - (def ^:const id-bigint (int 44)) - (def ^:const id-biginteger (int 45)) + (def ^:const id-byte (int 40)) + (def ^:const id-short (int 41)) + (def ^:const id-integer (int 42)) + (def ^:const id-long (int 43)) + (def ^:const id-bigint (int 44)) + (def ^:const id-biginteger (int 45)) - (def ^:const id-float (int 60)) - (def ^:const id-double (int 61)) - (def ^:const id-bigdec (int 62)) + (def ^:const id-float (int 60)) + (def ^:const id-double (int 61)) + (def ^:const id-bigdec (int 62)) - (def ^:const id-ratio (int 70)) + (def ^:const id-ratio (int 70)) - (def ^:const id-record (int 80)) - ;; (def ^:const id-type (int 81)) ; TODO? - (def ^:const id-prefixed-custom (int 82)) + (def ^:const id-record (int 80)) + ;; (def ^:const id-type (int 81)) ; TODO? + (def ^:const id-prefixed-custom (int 82)) - (def ^:const id-date (int 90)) - (def ^:const id-uuid (int 91)) + (def ^:const id-date (int 90)) + (def ^:const id-uuid (int 91)) ;;; Optimized, common-case types (v2.6+) - (def ^:const id-byte-as-long (int 100)) ; 1 vs 8 bytes - (def ^:const id-short-as-long (int 101)) ; 2 vs 8 bytes - (def ^:const id-int-as-long (int 102)) ; 4 vs 8 bytes - ;; (def ^:const id-compact-long (int 103)) ; 6->7 vs 8 bytes + (def ^:const id-byte-as-long (int 100)) ; 1 vs 8 bytes + (def ^:const id-short-as-long (int 101)) ; 2 vs 8 bytes + (def ^:const id-int-as-long (int 102)) ; 4 vs 8 bytes ;; - (def ^:const id-string-small (int 105)) ; 1 vs 4 byte length prefix - (def ^:const id-keyword-small (int 106)) ; '' + (def ^:const id-sm-string (int 105)) ; 1 vs 4 byte length prefix + (def ^:const id-sm-keyword (int 106)) ; '' ;; - ;; (def ^:const id-vector-small (int 110)) ; '' - ;; (def ^:const id-set-small (int 111)) ; '' - ;; (def ^:const id-map-small (int 112)) ; '' + ;; (def ^:const id-sm-vector (int 110)) ; '' + ;; (def ^:const id-sm-set (int 111)) ; '' + ;; (def ^:const id-sm-map (int 112)) ; '' ;;; DEPRECATED (old types will be supported only for thawing) - (def ^:const id-reader-depr1 (int 1)) ; v0.9.2+ for +64k support - (def ^:const id-string-depr1 (int 11)) ; v0.9.2+ for +64k support - (def ^:const id-map-depr1 (int 22)) ; v0.9.0+ for more efficient thaw - (def ^:const id-keyword-depr1 (int 12)) ; v2.0.0-alpha5+ for str consistecy - (def ^:const id-map-depr2 (int 27)) ; v2.11+ for count/2 - (def ^:const id-sorted-map-depr1 (int 29)) ; v2.11+ for count/2 + (def ^:const id-reader-depr1 (int 1)) ; v0.9.2+ for +64k support + (def ^:const id-string-depr1 (int 11)) ; v0.9.2+ for +64k support + (def ^:const id-map-depr1 (int 22)) ; v0.9.0+ for more efficient thaw + (def ^:const id-keyword-depr1 (int 12)) ; v2.0.0-alpha5+ for str consistecy + (def ^:const id-map-depr2 (int 27)) ; v2.11+ for count/2 + (def ^:const id-sorted-map-depr1 (int 29)) ; v2.11+ for count/2 ) ;;;; Ns imports (mostly for convenience of lib consumers) @@ -150,7 +151,7 @@ ;;;; Freezing (defprotocol Freezable - "Be careful about extending to interfaces, Ref. http://goo.gl/6gGRlU." + "Be careful about extending to interfaces, Ref. http://goo.gl/6gGRlU" (freeze-to-out* [this out])) (defmacro write-id [out id] `(.writeByte ~out ~id)) @@ -175,14 +176,8 @@ (let [x (with-meta x {:tag 'String})] `(write-bytes ~out (.getBytes ~x "UTF-8") ~small?))) -(defmacro write-compact-long "Uses 2->9 bytes." [out x] - `(write-bytes ~out (.toByteArray (java.math.BigInteger/valueOf (long ~x))) - :small)) - -(comment (alength (.toByteArray (java.math.BigInteger/valueOf Long/MAX_VALUE)))) - (defmacro ^:private freeze-to-out - "Like `freeze-to-out*` but with metadata support." + "Like `freeze-to-out*` but with metadata support" [out x] `(let [out# ~out, x# ~x] (when-let [m# (meta x#)] @@ -236,7 +231,7 @@ (freeze-to-out* [x ^DataOutput out] (let [ba (.getBytes x "UTF-8")] (if (<= (alength ^bytes ba) Byte/MAX_VALUE) - (do (write-id out id-string-small) + (do (write-id out id-sm-string) (write-bytes out ba :small)) (do (write-id out id-string) @@ -251,7 +246,7 @@ ba (.getBytes s "UTF-8")] (if (<= (alength ^bytes ba) Byte/MAX_VALUE) - (do (write-id out id-keyword-small) + (do (write-id out id-sm-keyword) (write-bytes out ba :small)) (do (write-id out id-keyword) @@ -386,8 +381,8 @@ [^bytes ba] (let [ba-len (alength ba)] (cond - ;; (> ba-len 4098) lzma2-compressor - ;; (> ba-len 2048) lz4hc-compressor + ;; (> ba-len 8192) lzma2-compressor + ;; (> ba-len 4098) lz4hc-compressor (> ba-len 1024) lz4-compressor :else nil))) @@ -396,7 +391,7 @@ default-freeze-compressor-selector) (defn set-default-freeze-compressor-selector! - "Sets root binding of `*default-freeze-compressor-selector*`." + "Sets root binding of `*default-freeze-compressor-selector*`" [selector] (alter-var-root #'*default-freeze-compressor-selector* (constantly selector))) @@ -460,8 +455,6 @@ (defmacro read-utf8 [in & [small?]] `(String. (read-bytes ~in ~small?) "UTF-8")) -(defmacro read-compact-long [in] `(long (BigInteger. (read-bytes ~in :small)))) - (defmacro ^:private read-coll [in coll] `(let [in# ~in] (encore/repeatedly-into ~coll (.readInt in#) (fn [] (thaw-from-in in#))))) @@ -551,8 +544,8 @@ id-keyword (keyword (read-utf8 in)) ;;; Optimized, common-case types (v2.6+) - id-string-small (read-utf8 in :small) - id-keyword-small (keyword (read-utf8 in :small)) + id-sm-string (read-utf8 in :small) + id-sm-keyword (keyword (read-utf8 in :small)) id-queue (read-coll in (PersistentQueue/EMPTY)) id-sorted-set (read-coll in (sorted-set)) @@ -577,7 +570,6 @@ id-byte-as-long (long (.readByte in)) id-short-as-long (long (.readShort in)) id-int-as-long (long (.readInt in)) - ;; id-compact-long (read-compact-long in) id-bigint (bigint (read-biginteger in)) id-biginteger (read-biginteger in) @@ -654,8 +646,8 @@ Nippy. To thaw custom types, extend the Clojure reader or see `extend-thaw`. Options include: - :compressor - An ICompressor, :auto (requires Nippy header), or nil. - :encryptor - An IEncryptor, :auto (requires Nippy header), or nil." + :compressor - An ICompressor, :auto (requires Nippy header), or nil + :encryptor - An IEncryptor, :auto (requires Nippy header), or nil" ([ba] (thaw ba nil)) ([^bytes ba @@ -737,8 +729,8 @@ (and (integer? custom-type-id) (<= 1 custom-type-id 128))))) (defn- coerce-custom-type-id - "* +ive byte id -> -ive byte id (for unprefixed custom types). - * Keyword id -> Short hash id (for prefixed custom types)." + "* +ive byte id -> -ive byte id (for unprefixed custom types) + * Keyword id -> Short hash id (for prefixed custom types)" [custom-type-id] (assert-custom-type-id custom-type-id) (if-not (keyword? custom-type-id) @@ -758,8 +750,8 @@ (defmacro extend-freeze "Extends Nippy to support freezing of a custom type (ideally concrete) with given id of form: - * Keyword - 2 byte overhead, resistent to id collisions. - * Byte ∈[1, 128] - no overhead, subject to id collisions. + * Keyword - 2 byte overhead, resistent to id collisions + * Integer ∈[1, 128] - no overhead, subject to id collisions (defrecord MyType [data]) (extend-freeze MyType :foo/my-type [x data-output] ; Keyword id @@ -809,7 +801,7 @@ ;;; Some useful custom types - EXPERIMENTAL ;; Mostly deprecated by :auto compressor selection -(defrecord Compressable-LZMA2 [value]) ; Why was this `LZMA2`, not `lzma2`? +(defrecord Compressable-LZMA2 [value]) ; Why was this `LZMA2` instead of `lzma2`? (extend-freeze Compressable-LZMA2 128 [x out] (let [ba (freeze (:value x) {:skip-header? true :compressor nil}) ba-len (alength ba) @@ -838,7 +830,7 @@ ;;;; Stress data (defrecord StressRecord [data]) -(def stress-data "Reference data used for tests & benchmarks." +(def stress-data "Reference data used for tests & benchmarks" (let [] {:bytes (byte-array [(byte 1) (byte 2) (byte 3)]) :nil nil @@ -899,18 +891,18 @@ :ex-info (ex-info "ExInfo" {:data "data"})})) (def stress-data-comparable - "Reference data with stuff removed that breaks roundtrip equality." + "Reference data with stuff removed that breaks roundtrip equality" (dissoc stress-data :bytes :throwable :exception :ex-info)) (def stress-data-benchable "Reference data with stuff removed that breaks reader or other utils we'll - be benching against." + be benching against" (dissoc stress-data :bytes :throwable :exception :ex-info :queue :queue-empty :byte :stress-record)) ;;;; Tools -(defn inspect-ba "Alpha - subject to change." +(defn inspect-ba "Alpha - subject to change" [ba & [thaw-opts]] (if-not (encore/bytes? ba) :not-ba (let [[first2bytes nextbytes] (encore/ba-split ba 2) From 327a800d8010931558c624aaaaf7a7cc5751ca7d Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 16:38:48 +0700 Subject: [PATCH 04/33] Experimental: optimize common case of small maps, sets, vectors --- src/taoensso/nippy.clj | 188 +++++++++++++++++++++++------------------ 1 file changed, 104 insertions(+), 84 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 16baa8f..e5f11db 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -120,9 +120,11 @@ (def ^:const id-sm-string (int 105)) ; 1 vs 4 byte length prefix (def ^:const id-sm-keyword (int 106)) ; '' ;; - ;; (def ^:const id-sm-vector (int 110)) ; '' - ;; (def ^:const id-sm-set (int 111)) ; '' - ;; (def ^:const id-sm-map (int 112)) ; '' + (def ^:const id-sm-vector (int 110)) ; '' + (def ^:const id-sm-set (int 111)) ; '' + (def ^:const id-sm-map (int 112)) ; '' + ;; + ;; TODO Additional optimizations (types) for 2-vecs and 3-vecs? ;;; DEPRECATED (old types will be supported only for thawing) (def ^:const id-reader-depr1 (int 1)) ; v0.9.2+ for +64k support @@ -154,19 +156,17 @@ "Be careful about extending to interfaces, Ref. http://goo.gl/6gGRlU" (freeze-to-out* [this out])) +(defn small-count? [n] (<= (long n) 127 #_Byte/MAX_VALUE)) (defmacro write-id [out id] `(.writeByte ~out ~id)) (defmacro write-bytes [out ba & [small?]] - (let [out (with-meta out {:tag 'java.io.DataOutput}) + (let [wc (if small? 'writeByte 'writeInt) + out (with-meta out {:tag 'java.io.DataOutput}) ba (with-meta ba {:tag 'bytes})] - (if small? ; Optimization, must be known before id's written - `(let [out# ~out, ba# ~ba - size# (alength ba#)] - (.writeByte out# (byte size#)) - (.write out# ba# 0 size#)) - `(let [out# ~out, ba# ~ba - size# (alength ba#)] - (.writeInt out# (int size#)) - (.write out# ba# 0 size#))))) + `(let [out# ~out + ba# ~ba + size# (alength ba#)] + (. out# ~wc size#) + (.write out# ba# 0 size#)))) (defmacro write-biginteger [out x] (let [x (with-meta x {:tag 'java.math.BigInteger})] @@ -185,6 +185,32 @@ (freeze-to-out* m# out#)) (freeze-to-out* x# out#))) +(defmacro write-coll [out x & [small?]] + (let [wc (if small? 'writeByte 'writeInt)] + `(if (counted? ~'x) + (do + (. ~'out ~wc (count ~'x)) + (encore/run!* (fn [i#] (freeze-to-out ~'out i#)) ~'x)) + (let [bas# (ByteArrayOutputStream. 64) + sout# (DataOutputStream. bas#) + cnt# (reduce (fn [^long cnt# i#] + (freeze-to-out sout# i#) + (unchecked-inc cnt#)) + 0 ~'x) + ba# (.toByteArray bas#)] + (. ~'out ~wc cnt#) + (.write ~'out ba# 0 (alength ba#)))))) + +(defmacro write-kvs [out x & [small?]] + (let [wc (if small? 'writeByte 'writeInt)] + `(do + (. ~'out ~wc (count ~'x)) + (encore/run-kv! + (fn [k# v#] + (freeze-to-out ~'out k#) + (freeze-to-out ~'out v#)) + ~'x)))) + (defmacro ^:private freezer [type id & body] `(extend-type ~type Freezable @@ -192,63 +218,57 @@ (write-id ~'out ~id) ~@body))) -(defmacro ^:private freezer-coll [type id & body] - `(freezer ~type ~id - (when-debug-mode - (when (instance? ISeq ~type) - (println (format "DEBUG - freezer-coll: %s for %s" ~type (type ~'x))))) - (if (counted? ~'x) - (do (.writeInt ~'out (count ~'x)) - (encore/run!* (fn [i#] (freeze-to-out ~'out i#)) ~'x)) - (let [bas# (ByteArrayOutputStream. 64) - sout# (DataOutputStream. bas#) - cnt# (reduce (fn [^long cnt# i#] - (freeze-to-out sout# i#) - (unchecked-inc cnt#)) - 0 ~'x) - ba# (.toByteArray bas#)] - (.writeInt ~'out cnt#) - (.write ~'out ba# 0 (alength ba#)))))) +(defmacro ^:private freezer-coll [type id & [id-sm]] + (if-not id-sm + `(freezer ~type ~id (write-coll ~'out ~'x)) + `(extend-type ~type + Freezable + (~'freeze-to-out* [~'x ~(with-meta 'out {:tag 'DataOutput})] + (if (small-count? (count ~'x)) + (do + (write-id ~'out ~id-sm) + (write-coll ~'out ~'x :small)) + (do + (write-id ~'out ~id) + (write-coll ~'out ~'x))))))) -(defmacro ^:private freezer-kvs [type id & body] - `(freezer ~type ~id - (.writeInt ~'out (count ~'x)) - (encore/run-kv! - (fn [k# v#] - (freeze-to-out ~'out k#) - (freeze-to-out ~'out v#)) - ~'x))) +(defmacro ^:private freezer-kvs [type id & [id-sm]] + (if-not id-sm + `(freezer ~type ~id (write-kvs ~'out ~'x)) + `(extend-type ~type + Freezable + (~'freeze-to-out* [~'x ~(with-meta 'out {:tag 'DataOutput})] + (if (small-count? (count ~'x)) + (do + (write-id ~'out ~id-sm) + (write-kvs ~'out ~'x :small)) + (do + (write-id ~'out ~id) + (write-kvs ~'out ~'x))))))) -(freezer (Class/forName "[B") id-bytes (write-bytes out ^bytes x)) +(freezer (Class/forName "[B") id-bytes (write-bytes out ^bytes x)) (freezer nil id-nil) -(freezer Boolean id-boolean (.writeBoolean out x)) +(freezer Boolean id-boolean (.writeBoolean out x)) +(freezer Character id-char (.writeChar out (int x))) -(freezer Character id-char (.writeChar out (int x))) -;; (freezer String id-string (write-utf8 out x)) - -(extend-type String ; Optimized common-case type +(extend-type String Freezable (freeze-to-out* [x ^DataOutput out] (let [ba (.getBytes x "UTF-8")] - (if (<= (alength ^bytes ba) Byte/MAX_VALUE) + (if (small-count? (alength ^bytes ba)) (do (write-id out id-sm-string) (write-bytes out ba :small)) - (do (write-id out id-string) (write-bytes out ba)))))) -(extend-type Keyword ; Optimized common-case type +(extend-type Keyword Freezable (freeze-to-out* [x ^DataOutput out] - (let [s (if-let [ns (namespace x)] - (str ns "/" (name x)) - (name x)) + (let [s (if-let [ns (namespace x)] (str ns "/" (name x)) (name x)) ba (.getBytes s "UTF-8")] - - (if (<= (alength ^bytes ba) Byte/MAX_VALUE) + (if (small-count? Byte/MAX_VALUE) (do (write-id out id-sm-keyword) (write-bytes out ba :small)) - (do (write-id out id-keyword) (write-bytes out ba)))))) @@ -256,9 +276,9 @@ (freezer-coll PersistentTreeSet id-sorted-set) (freezer-kvs PersistentTreeMap id-sorted-map) -(freezer-kvs APersistentMap id-map) -(freezer-coll APersistentVector id-vector) -(freezer-coll APersistentSet id-set) +(freezer-kvs APersistentMap id-map id-sm-map) +(freezer-coll APersistentVector id-vector id-sm-vector) +(freezer-coll APersistentSet id-set id-sm-set) (freezer-coll PersistentList id-list) ; No APersistentList (freezer-coll (type '()) id-list) @@ -270,11 +290,11 @@ (write-utf8 out (.getName (class x))) ; Reflect (freeze-to-out out (into {} x))) -(freezer Byte id-byte (.writeByte out x)) -(freezer Short id-short (.writeShort out x)) -(freezer Integer id-integer (.writeInt out x)) -;;(freezer Long id-long (.writeLong out x)) -(extend-type Long ; Optimized common-case type +(freezer Byte id-byte (.writeByte out x)) +(freezer Short id-short (.writeShort out x)) +(freezer Integer id-integer (.writeInt out x)) +;;(freezer Long id-long (.writeLong out x)) +(extend-type Long Freezable (freeze-to-out* [x ^DataOutput out] (let [^long x x] @@ -297,8 +317,6 @@ :else (do (write-id out id-long) (.writeLong out x)))))) -;; - (freezer BigInt id-bigint (write-biginteger out (.toBigInteger x))) (freezer BigInteger id-biginteger (write-biginteger out x)) @@ -439,14 +457,9 @@ (declare thaw-from-in) (defmacro read-bytes [in & [small?]] - (if small? ; Optimization, must be known before id's written + (let [rc (if small? 'readByte 'readInt)] `(let [in# ~in - size# (.readByte in#) - ba# (byte-array size#)] - (.readFully in# ba# 0 size#) - ba#) - `(let [in# ~in - size# (.readInt in#) + size# (. in# ~rc) ba# (byte-array size#)] (.readFully in# ba# 0 size#) ba#))) @@ -455,14 +468,17 @@ (defmacro read-utf8 [in & [small?]] `(String. (read-bytes ~in ~small?) "UTF-8")) -(defmacro ^:private read-coll [in coll] - `(let [in# ~in] (encore/repeatedly-into ~coll (.readInt in#) - (fn [] (thaw-from-in in#))))) +(defmacro ^:private read-coll [in coll & [small?]] + (let [rc (if small? 'readByte 'readInt)] + `(let [in# ~in] + (encore/repeatedly-into ~coll (. in# ~rc) + (fn [] (thaw-from-in in#)))))) -(defmacro ^:private read-kvs [in coll] - `(let [in# ~in] - (encore/repeatedly-into ~coll (.readInt in#) - (fn [] [(thaw-from-in in#) (thaw-from-in in#)])))) +(defmacro ^:private read-kvs [in coll & [small?]] + (let [rc (if small? 'readByte 'readInt)] + `(let [in# ~in] + (encore/repeatedly-into ~coll (. in# ~rc) + (fn [] [(thaw-from-in in#) (thaw-from-in in#)]))))) (defmacro ^:private read-kvs-depr1 [in coll] `(let [in# ~in] @@ -551,13 +567,17 @@ id-sorted-set (read-coll in (sorted-set)) id-sorted-map (read-kvs in (sorted-map)) - id-list (into '() (rseq (read-coll in []))) - id-vector (read-coll in []) - id-set (read-coll in #{}) - id-map (read-kvs in {}) - id-seq (or (seq (read-coll in [])) - (lazy-seq nil) ; Empty coll - ) + id-vector (read-coll in []) + id-sm-vector (read-coll in [] :small) + id-set (read-coll in #{}) + id-sm-set (read-coll in #{} :small) + id-map (read-kvs in {}) + id-sm-map (read-kvs in {} :small) + + id-list (into '() (rseq (read-coll in []))) + id-seq (or (seq (read-coll in [])) + (lazy-seq nil) ; Empty coll + ) id-meta (let [m (thaw-from-in in)] (with-meta (thaw-from-in in) m)) From fa17eb3a781a4acb65861df26cf81d65d888f44f Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 18:44:51 +0700 Subject: [PATCH 05/33] Update benchmarks --- src/taoensso/nippy/benchmarks.clj | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index a379e4a..10e82c1 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -66,6 +66,13 @@ ;; (bench {:reader? true :lzma2? true :fressian? true :laps 3}) ;; (bench {:laps 4}) ;; (bench {:laps 1 :lzma2? true}) + ;; (bench {:laps 1}) + + ;;; 2015 Sep 28, small collection optimizations + {:lzma2 {:round 56307, :freeze 36475, :thaw 19832, :size 11244}} + {:encrypted {:round 6062, :freeze 3802, :thaw 2260, :size 16148}} + {:default {:round 5482, :freeze 3382, :thaw 2100, :size 16128}} + {:fast {:round 4729, :freeze 2826, :thaw 1903, :size 16972}} ;;; 2015 Sep 29, various micro optimizations (incl. &arg elimination) {:reader {:round 63547, :freeze 19374, :thaw 44173, :size 27717}} From 89c9328596664ce57ef16e17536c07001d44574c Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 10:44:52 +0700 Subject: [PATCH 06/33] Experimental optimization: zero-copy `freeze` mode --- src/taoensso/nippy.clj | 55 +++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index e5f11db..e9e4c5d 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -378,7 +378,6 @@ (defn- wrap-header [data-ba head-meta] (if-let [head-ba (get-head-ba head-meta)] - ;; TODO Would be nice if we could avoid the array copy here: (encore/ba-concat head-ba data-ba) (throw (ex-info (format "Unrecognized header meta: %s" head-meta) {:head-meta head-meta})))) @@ -425,32 +424,44 @@ compressor (if legacy-mode? snappy-compressor compressor) encryptor (when password (if-not legacy-mode? encryptor nil)) skip-header? (or skip-header? legacy-mode?) + zero-copy-mode? (and (nil? compressor) (nil? encryptor)) baos (ByteArrayOutputStream. 64) dos (DataOutputStream. baos)] - (freeze-to-out! dos x) - (let [ba (.toByteArray baos) - compressor - (if (identical? compressor :auto) - (if skip-header? - lz4-compressor - (*default-freeze-compressor-selector* ba)) - (if (fn? compressor) - (compressor ba) ; Assume compressor selector fn - compressor ; Assume compressor - )) + (if zero-copy-mode? + (do ; Optimized case + (when-not skip-header? ; Avoid `wrap-header`'s array copy: + (let [head-ba (get-head-ba {:compressor-id nil :encryptor-id nil})] + (.write dos head-ba 0 4))) + (freeze-to-out! dos x) + (.toByteArray baos)) - ba (if compressor (compress compressor ba) ba) - ba (if encryptor (encrypt encryptor password ba) ba)] + (do + (freeze-to-out! dos x) + (let [ba (.toByteArray baos) - (if skip-header? ba - (wrap-header ba - {:compressor-id (when-let [c compressor] - (or (compression/standard-header-ids - (compression/header-id c)) :else)) - :encryptor-id (when-let [e encryptor] - (or (encryption/standard-header-ids - (encryption/header-id e)) :else))})))))) + compressor + (if (identical? compressor :auto) + (if skip-header? + lz4-compressor + (*default-freeze-compressor-selector* ba)) + (if (fn? compressor) + (compressor ba) ; Assume compressor selector fn + compressor ; Assume compressor + )) + + ba (if compressor (compress compressor ba) ba) + ba (if encryptor (encrypt encryptor password ba) ba)] + + (if skip-header? + ba + (wrap-header ba + {:compressor-id (when-let [c compressor] + (or (compression/standard-header-ids + (compression/header-id c)) :else)) + :encryptor-id (when-let [e encryptor] + (or (encryption/standard-header-ids + (encryption/header-id e)) :else))})))))))) ;;;; Thawing From da77b3d5826380288bd133ceb2eccd29374e87a9 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 11:33:49 +0700 Subject: [PATCH 07/33] NB: Remove (long-deprecated) freezing legacy mode --- src/taoensso/nippy.clj | 7 ++----- test/taoensso/nippy/tests/main.clj | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index e9e4c5d..89eaa34 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -27,7 +27,7 @@ ;; { * 1-byte type id. ;; * Arb-length payload. } ... ;; -;; [1] Inclusion of header is strongly recommended. Purpose: +;; [1] Inclusion of header is *strongly* recommended. Purpose: ;; * Sanity check (confirm that data appears to be Nippy data) ;; * Nippy version check (=> supports changes to data schema over time) ;; * Supports :auto thaw compressor, encryptor @@ -420,10 +420,7 @@ :or {compressor :auto encryptor aes128-encryptor} :as opts}] - (let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze - compressor (if legacy-mode? snappy-compressor compressor) - encryptor (when password (if-not legacy-mode? encryptor nil)) - skip-header? (or skip-header? legacy-mode?) + (let [encryptor (when password encryptor) zero-copy-mode? (and (nil? compressor) (nil? encryptor)) baos (ByteArrayOutputStream. 64) dos (DataOutputStream. baos)] diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index d060abf..a05e17a 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -17,8 +17,8 @@ (expect (do (println (str "Clojure version: " *clojure-version*)) true)) (expect test-data ((comp thaw freeze) test-data)) -(expect test-data ((comp #(thaw % {}) - #(freeze % {:legacy-mode true})) +(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor}) + #(freeze % {:skip-header? true})) test-data)) (expect test-data ((comp #(thaw % {:password [:salted "p"]}) #(freeze % {:password [:salted "p"]})) From 9c1e8751c4fe505b8608af5117275f708b88c6d7 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 14:18:21 +0700 Subject: [PATCH 08/33] Simplify stream freeze API, switch from macros->fns --- src/taoensso/nippy.clj | 51 +++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 89eaa34..7e2d346 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -154,7 +154,7 @@ (defprotocol Freezable "Be careful about extending to interfaces, Ref. http://goo.gl/6gGRlU" - (freeze-to-out* [this out])) + (-freeze-to-out [this out])) (defn small-count? [n] (<= (long n) 127 #_Byte/MAX_VALUE)) (defmacro write-id [out id] `(.writeByte ~out ~id)) @@ -176,25 +176,25 @@ (let [x (with-meta x {:tag 'String})] `(write-bytes ~out (.getBytes ~x "UTF-8") ~small?))) -(defmacro ^:private freeze-to-out - "Like `freeze-to-out*` but with metadata support" - [out x] - `(let [out# ~out, x# ~x] - (when-let [m# (meta x#)] - (write-id out# ~id-meta) - (freeze-to-out* m# out#)) - (freeze-to-out* x# out#))) +(defn freeze-to-out! + "Serializes arg (any Clojure data type) to a DataOutput" + ;; Basically just wraps `-freeze-to-out` with different arg order + metadata support + [^DataOutput data-output x] + (when-let [m (meta x)] + (write-id data-output id-meta) + (-freeze-to-out m data-output)) + (-freeze-to-out x data-output)) (defmacro write-coll [out x & [small?]] (let [wc (if small? 'writeByte 'writeInt)] `(if (counted? ~'x) (do (. ~'out ~wc (count ~'x)) - (encore/run!* (fn [i#] (freeze-to-out ~'out i#)) ~'x)) + (encore/run!* (fn [i#] (freeze-to-out! ~'out i#)) ~'x)) (let [bas# (ByteArrayOutputStream. 64) sout# (DataOutputStream. bas#) cnt# (reduce (fn [^long cnt# i#] - (freeze-to-out sout# i#) + (freeze-to-out! sout# i#) (unchecked-inc cnt#)) 0 ~'x) ba# (.toByteArray bas#)] @@ -207,14 +207,14 @@ (. ~'out ~wc (count ~'x)) (encore/run-kv! (fn [k# v#] - (freeze-to-out ~'out k#) - (freeze-to-out ~'out v#)) + (freeze-to-out! ~'out k#) + (freeze-to-out! ~'out v#)) ~'x)))) (defmacro ^:private freezer [type id & body] `(extend-type ~type Freezable - (~'freeze-to-out* [~'x ~(with-meta 'out {:tag 'DataOutput})] + (~'-freeze-to-out [~'x ~(with-meta 'out {:tag 'DataOutput})] (write-id ~'out ~id) ~@body))) @@ -223,7 +223,7 @@ `(freezer ~type ~id (write-coll ~'out ~'x)) `(extend-type ~type Freezable - (~'freeze-to-out* [~'x ~(with-meta 'out {:tag 'DataOutput})] + (~'-freeze-to-out [~'x ~(with-meta 'out {:tag 'DataOutput})] (if (small-count? (count ~'x)) (do (write-id ~'out ~id-sm) @@ -237,7 +237,7 @@ `(freezer ~type ~id (write-kvs ~'out ~'x)) `(extend-type ~type Freezable - (~'freeze-to-out* [~'x ~(with-meta 'out {:tag 'DataOutput})] + (~'-freeze-to-out [~'x ~(with-meta 'out {:tag 'DataOutput})] (if (small-count? (count ~'x)) (do (write-id ~'out ~id-sm) @@ -253,7 +253,7 @@ (extend-type String Freezable - (freeze-to-out* [x ^DataOutput out] + (-freeze-to-out [x ^DataOutput out] (let [ba (.getBytes x "UTF-8")] (if (small-count? (alength ^bytes ba)) (do (write-id out id-sm-string) @@ -263,7 +263,7 @@ (extend-type Keyword Freezable - (freeze-to-out* [x ^DataOutput out] + (-freeze-to-out [x ^DataOutput out] (let [s (if-let [ns (namespace x)] (str ns "/" (name x)) (name x)) ba (.getBytes s "UTF-8")] (if (small-count? Byte/MAX_VALUE) @@ -288,7 +288,7 @@ (freezer IRecord id-record (write-utf8 out (.getName (class x))) ; Reflect - (freeze-to-out out (into {} x))) + (freeze-to-out! out (into {} x))) (freezer Byte id-byte (.writeByte out x)) (freezer Short id-short (.writeShort out x)) @@ -296,7 +296,7 @@ ;;(freezer Long id-long (.writeLong out x)) (extend-type Long Freezable - (freeze-to-out* [x ^DataOutput out] + (-freeze-to-out [x ^DataOutput out] (let [^long x x] (cond (and (<= x #_Byte/MAX_VALUE 127) @@ -337,7 +337,7 @@ (def ^:dynamic *final-freeze-fallback* "Alpha - subject to change." nil) (defn freeze-fallback-as-str "Alpha-subject to change." [x out] - (freeze-to-out* {:nippy/unfreezable (encore/pr-edn x) :type (type x)} out)) + (-freeze-to-out {:nippy/unfreezable (encore/pr-edn x) :type (type x)} out)) (comment (require '[clojure.core.async :as async]) @@ -348,7 +348,7 @@ ;; interfering with higher-level implementations, Ref. http://goo.gl/6f7SKl (extend-type Object Freezable - (freeze-to-out* [x ^DataOutput out] + (-freeze-to-out [x ^DataOutput out] (cond (utils/serializable? x) ; Fallback #1: Java's Serializable interface (do (when-debug-mode @@ -385,11 +385,6 @@ (comment (wrap-header (.getBytes "foo") {:compressor-id :lz4 :encryptor-id nil})) -(defn freeze-to-out! - "Low-level API. Serializes arg (any Clojure data type) to a DataOutput." - [^DataOutput data-output x] - (freeze-to-out data-output x)) - (defn default-freeze-compressor-selector "Strategy: * Prioritize speed, but allow lz4. @@ -790,7 +785,7 @@ [type custom-type-id [x out] & body] (assert-custom-type-id custom-type-id) `(extend-type ~type Freezable - (~'freeze-to-out* [~x ~(with-meta out {:tag 'java.io.DataOutput})] + (~'-freeze-to-out [~x ~(with-meta out {:tag 'java.io.DataOutput})] (if-not ~(keyword? custom-type-id) ;; Unprefixed [cust byte id][payload]: (write-id ~out ~(coerce-custom-type-id custom-type-id)) From f67f9da64e8354deb851f2075bd6bf4e101cdedd Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 14:26:05 +0700 Subject: [PATCH 09/33] Remove alpha status on `final-freeze-fallback` --- src/taoensso/nippy.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 7e2d346..9648ac1 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -335,8 +335,8 @@ (.writeLong out (.getMostSignificantBits x)) (.writeLong out (.getLeastSignificantBits x))) -(def ^:dynamic *final-freeze-fallback* "Alpha - subject to change." nil) -(defn freeze-fallback-as-str "Alpha-subject to change." [x out] +(def ^:dynamic *final-freeze-fallback* nil) +(defn freeze-fallback-as-str [out x] (-freeze-to-out {:nippy/unfreezable (encore/pr-edn x) :type (type x)} out)) (comment @@ -364,7 +364,8 @@ (write-utf8 out (encore/pr-edn x))) :else ; Fallback #3: *final-freeze-fallback* - (if-let [ffb *final-freeze-fallback*] (ffb x out) + (if-let [ffb *final-freeze-fallback*] + (ffb x out) (throw (ex-info (format "Unfreezable type: %s %s" (type x) (str x)) {:type (type x) :as-str (encore/pr-edn x)})))))) From 734e88b20cb740bd9ee9ddfeaae0a8cd7ec1a04e Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 14:26:54 +0700 Subject: [PATCH 10/33] `defonce` on all dynamic vars (allow `alter-var-root`) --- src/taoensso/nippy.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 9648ac1..dbebf95 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -335,7 +335,7 @@ (.writeLong out (.getMostSignificantBits x)) (.writeLong out (.getLeastSignificantBits x))) -(def ^:dynamic *final-freeze-fallback* nil) +(encore/defonce* ^:dynamic *final-freeze-fallback* nil) (defn freeze-fallback-as-str [out x] (-freeze-to-out {:nippy/unfreezable (encore/pr-edn x) :type (type x)} out)) @@ -491,7 +491,7 @@ (def ^:private class-method-sig (into-array Class [IPersistentMap])) -(def ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) +(encore/defonce* ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) (defn swap-custom-readers! [f] (alter-var-root #'*custom-readers* f)) (defn- read-custom! [type-id in] From 50ffb78c228c0f2a34f6f43f1e0fd987eff84f59 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 14:30:25 +0700 Subject: [PATCH 11/33] Refer rename: `encore`->`enc` --- src/taoensso/nippy.clj | 76 +++++++++++++++--------------- src/taoensso/nippy/benchmarks.clj | 10 ++-- src/taoensso/nippy/compression.clj | 8 ++-- src/taoensso/nippy/encryption.clj | 22 ++++----- src/taoensso/nippy/utils.clj | 6 +-- 5 files changed, 61 insertions(+), 61 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index dbebf95..c56639e 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -2,7 +2,7 @@ "High-performance JVM Clojure serialization library. Originally adapted from Deep-Freeze (https://goo.gl/OePPGr)." {:author "Peter Taoussanis (@ptaoussanis)"} - (:require [taoensso.encore :as encore] + (:require [taoensso.encore :as enc] [taoensso.nippy (utils :as utils) (compression :as compression) @@ -18,9 +18,9 @@ PersistentQueue PersistentTreeMap PersistentTreeSet PersistentList ; LazySeq IRecord ISeq])) -(if (vector? taoensso.encore/encore-version) - (encore/assert-min-encore-version [2 16 0]) - (encore/assert-min-encore-version 2.16)) +(if (vector? enc/encore-version) + (enc/assert-min-encore-version [2 16 0]) + (enc/assert-min-encore-version 2.16)) ;;;; Nippy data format ;; * 4-byte header (Nippy v2.x+) (may be disabled but incl. by default) [1] @@ -137,18 +137,18 @@ ;;;; Ns imports (mostly for convenience of lib consumers) -(encore/defalias compress compression/compress) -(encore/defalias decompress compression/decompress) -(encore/defalias snappy-compressor compression/snappy-compressor) -(encore/defalias lzma2-compressor compression/lzma2-compressor) -(encore/defalias lz4-compressor compression/lz4-compressor) -(encore/defalias lz4hc-compressor compression/lz4hc-compressor) +(enc/defalias compress compression/compress) +(enc/defalias decompress compression/decompress) +(enc/defalias snappy-compressor compression/snappy-compressor) +(enc/defalias lzma2-compressor compression/lzma2-compressor) +(enc/defalias lz4-compressor compression/lz4-compressor) +(enc/defalias lz4hc-compressor compression/lz4hc-compressor) -(encore/defalias encrypt encryption/encrypt) -(encore/defalias decrypt encryption/decrypt) -(encore/defalias aes128-encryptor encryption/aes128-encryptor) +(enc/defalias encrypt encryption/encrypt) +(enc/defalias decrypt encryption/decrypt) +(enc/defalias aes128-encryptor encryption/aes128-encryptor) -(encore/defalias freezable? utils/freezable?) +(enc/defalias freezable? utils/freezable?) ;;;; Freezing @@ -190,7 +190,7 @@ `(if (counted? ~'x) (do (. ~'out ~wc (count ~'x)) - (encore/run!* (fn [i#] (freeze-to-out! ~'out i#)) ~'x)) + (enc/run!* (fn [i#] (freeze-to-out! ~'out i#)) ~'x)) (let [bas# (ByteArrayOutputStream. 64) sout# (DataOutputStream. bas#) cnt# (reduce (fn [^long cnt# i#] @@ -205,7 +205,7 @@ (let [wc (if small? 'writeByte 'writeInt)] `(do (. ~'out ~wc (count ~'x)) - (encore/run-kv! + (enc/run-kv! (fn [k# v#] (freeze-to-out! ~'out k#) (freeze-to-out! ~'out v#)) @@ -335,9 +335,9 @@ (.writeLong out (.getMostSignificantBits x)) (.writeLong out (.getLeastSignificantBits x))) -(encore/defonce* ^:dynamic *final-freeze-fallback* nil) +(enc/defonce* ^:dynamic *final-freeze-fallback* nil) (defn freeze-fallback-as-str [out x] - (-freeze-to-out {:nippy/unfreezable (encore/pr-edn x) :type (type x)} out)) + (-freeze-to-out {:nippy/unfreezable (enc/pr-edn x) :type (type x)} out)) (comment (require '[clojure.core.async :as async]) @@ -361,25 +361,25 @@ (do (when-debug-mode (println (format "DEBUG - Reader fallback: %s" (type x)))) (write-id out id-reader) - (write-utf8 out (encore/pr-edn x))) + (write-utf8 out (enc/pr-edn x))) :else ; Fallback #3: *final-freeze-fallback* (if-let [ffb *final-freeze-fallback*] (ffb x out) (throw (ex-info (format "Unfreezable type: %s %s" (type x) (str x)) {:type (type x) - :as-str (encore/pr-edn x)})))))) + :as-str (enc/pr-edn x)})))))) (def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta)) (def ^:private get-head-ba (memoize (fn [head-meta] (when-let [meta-id (get head-meta-id (assoc head-meta :version head-version))] - (encore/ba-concat head-sig (byte-array [meta-id])))))) + (enc/ba-concat head-sig (byte-array [meta-id])))))) (defn- wrap-header [data-ba head-meta] (if-let [head-ba (get-head-ba head-meta)] - (encore/ba-concat head-ba data-ba) + (enc/ba-concat head-ba data-ba) (throw (ex-info (format "Unrecognized header meta: %s" head-meta) {:head-meta head-meta})))) @@ -399,7 +399,7 @@ (> ba-len 1024) lz4-compressor :else nil))) -(encore/defonce* ^:dynamic *default-freeze-compressor-selector* +(enc/defonce* ^:dynamic *default-freeze-compressor-selector* "(fn selector [^bytes ba])->compressor used by `(freeze {:compressor :auto})." default-freeze-compressor-selector) @@ -475,23 +475,23 @@ (defmacro ^:private read-coll [in coll & [small?]] (let [rc (if small? 'readByte 'readInt)] `(let [in# ~in] - (encore/repeatedly-into ~coll (. in# ~rc) + (enc/repeatedly-into ~coll (. in# ~rc) (fn [] (thaw-from-in in#)))))) (defmacro ^:private read-kvs [in coll & [small?]] (let [rc (if small? 'readByte 'readInt)] `(let [in# ~in] - (encore/repeatedly-into ~coll (. in# ~rc) + (enc/repeatedly-into ~coll (. in# ~rc) (fn [] [(thaw-from-in in#) (thaw-from-in in#)]))))) (defmacro ^:private read-kvs-depr1 [in coll] `(let [in# ~in] - (encore/repeatedly-into ~coll (quot (.readInt in#) 2) + (enc/repeatedly-into ~coll (quot (.readInt in#) 2) (fn [] [(thaw-from-in in#) (thaw-from-in in#)])))) (def ^:private class-method-sig (into-array Class [IPersistentMap])) -(encore/defonce* ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) +(enc/defonce* ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) (defn swap-custom-readers! [f] (alter-var-root #'*custom-readers* f)) (defn- read-custom! [type-id in] @@ -516,12 +516,12 @@ (when-debug-mode (println (format "DEBUG - thawing type-id: %s" type-id))) - (encore/case-eval type-id + (enc/case-eval type-id id-reader (let [edn (read-utf8 in)] (try - (encore/read-edn {:readers *data-readers*} edn) + (enc/read-edn {:readers *data-readers*} edn) (catch Exception e {:type :reader :throwable e @@ -615,9 +615,9 @@ ;;; DEPRECATED id-sorted-map-depr1 (read-kvs-depr1 in (sorted-map)) id-map-depr2 (read-kvs-depr1 in {}) - id-reader-depr1 (encore/read-edn (.readUTF in)) + id-reader-depr1 (enc/read-edn (.readUTF in)) id-string-depr1 (.readUTF in) - id-map-depr1 (apply hash-map (encore/repeatedly-into [] (* 2 (.readInt in)) + id-map-depr1 (apply hash-map (enc/repeatedly-into [] (* 2 (.readInt in)) (fn [] (thaw-from-in in)))) id-keyword-depr1 (keyword (.readUTF in)) @@ -639,9 +639,9 @@ (thaw-from-in data-input)) (defn- try-parse-header [ba] - (when-let [[head-ba data-ba] (encore/ba-split ba 4)] - (let [[head-sig* [meta-id]] (encore/ba-split head-ba 3)] - (when (encore/ba= head-sig* head-sig) ; Header appears to be well-formed + (when-let [[head-ba data-ba] (enc/ba-split ba 4)] + (let [[head-sig* [meta-id]] (enc/ba-split head-ba 3)] + (when (enc/ba= head-sig* head-sig) ; Header appears to be well-formed [data-ba (get head-meta meta-id {:unrecognized-meta? true})])))) (defn- get-auto-compressor [compressor-id] @@ -928,12 +928,12 @@ (defn inspect-ba "Alpha - subject to change" [ba & [thaw-opts]] - (if-not (encore/bytes? ba) :not-ba - (let [[first2bytes nextbytes] (encore/ba-split ba 2) + (if-not (enc/bytes? ba) :not-ba + (let [[first2bytes nextbytes] (enc/ba-split ba 2) known-wrapper (cond - (encore/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin - (encore/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj) + (enc/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin + (enc/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj) unwrapped-ba (if known-wrapper nextbytes ba) [data-ba nippy-header] (or (try-parse-header unwrapped-ba) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 10e82c1..010b446 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,7 +1,7 @@ (ns taoensso.nippy.benchmarks {:author "Peter Taoussanis"} - (:require [clojure.data.fressian :as fressian] - [taoensso.encore :as encore] + (:require [clojure.data.fressian :as fressian] + [taoensso.encore :as enc] [taoensso.nippy :as nippy :refer (freeze thaw)])) (def data nippy/stress-data-benchable) @@ -19,7 +19,7 @@ (comment (fressian-thaw (fressian-freeze data))) -(defmacro bench* [& body] `(encore/bench 10000 {:warmup-laps 20000} ~@body)) +(defmacro bench* [& body] `(enc/bench 10000 {:warmup-laps 20000} ~@body)) (defn bench1 [freezer thawer & [sizer]] (let [data-frozen (freezer data) time-freeze (bench* (freezer data)) @@ -36,7 +36,7 @@ (println (str "\nLap " (inc l) "/" laps "...")) (when reader? ; Slow - (println {:reader (bench1 encore/pr-edn encore/read-edn + (println {:reader (bench1 enc/pr-edn enc/read-edn #(count (.getBytes ^String % "UTF-8")))})) (println {:default (bench1 #(freeze % {}) @@ -58,7 +58,7 @@ (println "\nDone! (Time for cake?)") true) -(comment (encore/read-edn (encore/pr-edn data)) +(comment (enc/read-edn (enc/pr-edn data)) (bench1 fressian-freeze fressian-thaw)) (comment diff --git a/src/taoensso/nippy/compression.clj b/src/taoensso/nippy/compression.clj index 195e4a9..1fb8917 100644 --- a/src/taoensso/nippy/compression.clj +++ b/src/taoensso/nippy/compression.clj @@ -1,6 +1,6 @@ (ns taoensso.nippy.compression {:author "Peter Taoussanis"} - (:require [taoensso.encore :as encore]) + (:require [taoensso.encore :as enc]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream DataOutputStream])) @@ -122,10 +122,10 @@ (comment (def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8")) (defn bench1 [compressor] - {:time (encore/bench 10000 {:nlaps-warmup 10000} + {:time (enc/bench 10000 {:nlaps-warmup 10000} (->> ba-bench (compress compressor) (decompress compressor))) - :ratio (encore/round2 (/ (count (compress compressor ba-bench)) - (count ba-bench)))}) + :ratio (enc/round2 (/ (count (compress compressor ba-bench)) + (count ba-bench)))}) (println {:snappy (bench1 snappy-compressor) diff --git a/src/taoensso/nippy/encryption.clj b/src/taoensso/nippy/encryption.clj index 03088d8..eb946dc 100644 --- a/src/taoensso/nippy/encryption.clj +++ b/src/taoensso/nippy/encryption.clj @@ -2,7 +2,7 @@ "Simple no-nonsense crypto with reasonable defaults. Because your Clojure data deserves some privacy." {:author "Peter Taoussanis"} - (:require [taoensso.encore :as encore])) + (:require [taoensso.encore :as enc])) ;;;; Interface @@ -16,15 +16,15 @@ ;;;; Default digests, ciphers, etc. (def ^:private aes128-cipher* - (encore/thread-local-proxy + (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))) (def ^:private sha512-md* - (encore/thread-local-proxy + (enc/thread-local-proxy (java.security.MessageDigest/getInstance "SHA-512"))) (def ^:private prng* - (encore/thread-local-proxy + (enc/thread-local-proxy (java.security.SecureRandom/getInstance "SHA1PRNG"))) (defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*)) @@ -44,7 +44,7 @@ [salt-ba ^String pwd] (let [md (sha512-md)] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] - (if salt-ba (encore/ba-concat salt-ba pwd-ba) pwd-ba)) + (if salt-ba (enc/ba-concat salt-ba pwd-ba) pwd-ba)) n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] (if-not (zero? n) (recur (.digest md ba) (dec n)) @@ -82,26 +82,26 @@ salt? (identical? type :salted) iv-ba (rand-bytes aes128-block-size) salt-ba (when salt? (rand-bytes salt-size)) - prefix-ba (if-not salt? iv-ba (encore/ba-concat iv-ba salt-ba)) + prefix-ba (if-not salt? iv-ba (enc/ba-concat iv-ba salt-ba)) key (if salt? (key-gen salt-ba pwd) - (encore/memoized key-cache key-gen salt-ba pwd)) + (enc/memoized key-cache key-gen salt-ba pwd)) iv (javax.crypto.spec.IvParameterSpec. iv-ba) cipher (aes128-cipher)] (.init cipher javax.crypto.Cipher/ENCRYPT_MODE ^javax.crypto.spec.SecretKeySpec key iv) - (encore/ba-concat prefix-ba (.doFinal cipher data-ba)))) + (enc/ba-concat prefix-ba (.doFinal cipher data-ba)))) (decrypt [_ typed-pwd ba] (let [[type pwd] (destructure-typed-pwd typed-pwd) salt? (= type :salted) prefix-size (+ aes128-block-size (if salt? salt-size 0)) - [prefix-ba data-ba] (encore/ba-split ba prefix-size) + [prefix-ba data-ba] (enc/ba-split ba prefix-size) [iv-ba salt-ba] (if-not salt? [prefix-ba nil] - (encore/ba-split prefix-ba aes128-block-size)) + (enc/ba-split prefix-ba aes128-block-size)) key (if salt? (key-gen salt-ba pwd) - (encore/memoized key-cache key-gen salt-ba pwd)) + (enc/memoized key-cache key-gen salt-ba pwd)) iv (javax.crypto.spec.IvParameterSpec. iv-ba) cipher (aes128-cipher)] (.init cipher javax.crypto.Cipher/DECRYPT_MODE diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index aa265ea..7dfcc91 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -1,7 +1,7 @@ (ns taoensso.nippy.utils {:author "Peter Taoussanis"} (:require [clojure.string :as str] - [taoensso.encore :as encore]) + [taoensso.encore :as enc]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable ObjectOutputStream ObjectInputStream])) @@ -17,7 +17,7 @@ cacheable? (not (re-find #"__\d+" (str t))) ; gensym form test (fn [] (try (f-test x) (catch Exception _ false)))] (if-not cacheable? (test) - @(encore/swap-val! cache t #(if % % (delay (test))))))))) + @(enc/swap-val! cache t #(if % % (delay (test))))))))) (def serializable? (memoize-type-test @@ -33,7 +33,7 @@ (cast class object) true))))) -(def readable? (memoize-type-test (fn [x] (-> x encore/pr-edn encore/read-edn) true))) +(def readable? (memoize-type-test (fn [x] (-> x enc/pr-edn enc/read-edn) true))) (comment (serializable? "Hello world") From 15f0de1658f94cc0ffe7a266614aa6a4b4a2cdba Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 14:36:23 +0700 Subject: [PATCH 12/33] Simplify stream thaw API, switch from macros->fns --- src/taoensso/nippy.clj | 94 +++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index c56639e..28a6e22 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -458,36 +458,41 @@ ;;;; Thawing -(declare thaw-from-in) +(declare thaw-from-in!) -(defmacro read-bytes [in & [small?]] - (let [rc (if small? 'readByte 'readInt)] - `(let [in# ~in - size# (. in# ~rc) - ba# (byte-array size#)] - (.readFully in# ba# 0 size#) - ba#))) +(defn read-bytes ^bytes [^DataInput in] + (let [size (.readInt in) + ba (byte-array size)] + (.readFully in ba 0 size) + ba)) -(defmacro read-biginteger [in] `(BigInteger. (read-bytes ~in))) -(defmacro read-utf8 [in & [small?]] - `(String. (read-bytes ~in ~small?) "UTF-8")) +(defn read-sm-bytes ^bytes [^DataInput in] + (let [size (.readByte in) + ba (byte-array size)] + (.readFully in ba 0 size) + ba)) -(defmacro ^:private read-coll [in coll & [small?]] - (let [rc (if small? 'readByte 'readInt)] - `(let [in# ~in] - (enc/repeatedly-into ~coll (. in# ~rc) - (fn [] (thaw-from-in in#)))))) +(defn read-biginteger ^BigInteger [^DataInput in] (BigInteger. (read-bytes in))) +(defn read-utf8 ^String [^DataInput in] (String. (read-bytes in) "UTF-8")) +(defn read-sm-utf8 ^String [^DataInput in] (String. (read-sm-bytes in) "UTF-8")) -(defmacro ^:private read-kvs [in coll & [small?]] - (let [rc (if small? 'readByte 'readInt)] - `(let [in# ~in] - (enc/repeatedly-into ~coll (. in# ~rc) - (fn [] [(thaw-from-in in#) (thaw-from-in in#)]))))) +(defn read-coll [^DataInput in to-coll] + (enc/repeatedly-into to-coll (.readInt in) (fn [] (thaw-from-in! in)))) -(defmacro ^:private read-kvs-depr1 [in coll] - `(let [in# ~in] - (enc/repeatedly-into ~coll (quot (.readInt in#) 2) - (fn [] [(thaw-from-in in#) (thaw-from-in in#)])))) +(defn read-sm-coll [^DataInput in to-coll] + (enc/repeatedly-into to-coll (.readByte in) (fn [] (thaw-from-in! in)))) + +(defn read-kvs [^DataInput in to-coll] + (enc/repeatedly-into to-coll (.readInt in) + (fn [] [(thaw-from-in! in) (thaw-from-in! in)]))) + +(defn read-sm-kvs [^DataInput in to-coll] + (enc/repeatedly-into to-coll (.readByte in) + (fn [] [(thaw-from-in! in) (thaw-from-in! in)]))) + +(defn read-kvs-depr1 [^DataInput in to-coll] + (enc/repeatedly-into to-coll (quot (.readInt in) 2) + (fn [] [(thaw-from-in! in) (thaw-from-in! in)]))) (def ^:private class-method-sig (into-array Class [IPersistentMap])) @@ -509,9 +514,12 @@ type-id) {:internal-type-id type-id})))) -(defn- thaw-from-in - [^DataInput in] - (let [type-id (.readByte in)] +(defn thaw-from-in! + "Deserializes a frozen object from given DataInput to its original Clojure + data type" + [^DataInput data-input] + (let [in data-input + type-id (.readByte in)] (try (when-debug-mode (println (format "DEBUG - thawing type-id: %s" type-id))) @@ -544,8 +552,8 @@ :nippy/unthawable {:class-name class-name :content nil}}))) id-record - (let [class-name (read-utf8 in) - content (thaw-from-in in)] + (let [class-name (read-utf8 in) + content (thaw-from-in! in)] (try (let [class (Class/forName class-name) method (.getMethod class "create" class-method-sig)] @@ -564,26 +572,26 @@ id-keyword (keyword (read-utf8 in)) ;;; Optimized, common-case types (v2.6+) - id-sm-string (read-utf8 in :small) - id-sm-keyword (keyword (read-utf8 in :small)) + id-sm-string (read-sm-utf8 in) + id-sm-keyword (keyword (read-sm-utf8 in)) id-queue (read-coll in (PersistentQueue/EMPTY)) id-sorted-set (read-coll in (sorted-set)) id-sorted-map (read-kvs in (sorted-map)) - id-vector (read-coll in []) - id-sm-vector (read-coll in [] :small) - id-set (read-coll in #{}) - id-sm-set (read-coll in #{} :small) - id-map (read-kvs in {}) - id-sm-map (read-kvs in {} :small) + id-vector (read-coll in []) + id-sm-vector (read-sm-coll in []) + id-set (read-coll in #{}) + id-sm-set (read-sm-coll in #{}) + id-map (read-kvs in {}) + id-sm-map (read-sm-kvs in {}) id-list (into '() (rseq (read-coll in []))) id-seq (or (seq (read-coll in [])) (lazy-seq nil) ; Empty coll ) - id-meta (let [m (thaw-from-in in)] (with-meta (thaw-from-in in) m)) + id-meta (let [m (thaw-from-in! in)] (with-meta (thaw-from-in! in) m)) id-byte (.readByte in) id-short (.readShort in) @@ -618,7 +626,7 @@ id-reader-depr1 (enc/read-edn (.readUTF in)) id-string-depr1 (.readUTF in) id-map-depr1 (apply hash-map (enc/repeatedly-into [] (* 2 (.readInt in)) - (fn [] (thaw-from-in in)))) + (fn [] (thaw-from-in! in)))) id-keyword-depr1 (keyword (.readUTF in)) id-prefixed-custom ; Prefixed custom type @@ -632,12 +640,6 @@ (throw (ex-info (format "Thaw failed against type-id: %s" type-id) {:type-id type-id} e)))))) -(defn thaw-from-in! - "Low-level API. Deserializes a frozen object from given DataInput to its - original Clojure data type." - [data-input] - (thaw-from-in data-input)) - (defn- try-parse-header [ba] (when-let [[head-ba data-ba] (enc/ba-split ba 4)] (let [[head-sig* [meta-id]] (enc/ba-split head-ba 3)] From 998dabc1957fa1ce539bd15568c828c7fdd9bb76 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 16:02:46 +0700 Subject: [PATCH 13/33] NB: refactor freezing utils for easier use by libs + custom extensions, etc. --- src/taoensso/nippy.clj | 414 ++++++++++++++--------------- src/taoensso/nippy/benchmarks.clj | 8 +- test/taoensso/nippy/tests/main.clj | 3 +- 3 files changed, 204 insertions(+), 221 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 28a6e22..464eded 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -61,8 +61,7 @@ ;;;; Data type IDs -(do ; Just for easier IDE collapsing - +(do ;; ** Negative ids reserved for user-defined types ** ;; (def ^:const id-reserved (int 0)) @@ -137,44 +136,75 @@ ;;;; Ns imports (mostly for convenience of lib consumers) -(enc/defalias compress compression/compress) -(enc/defalias decompress compression/decompress) -(enc/defalias snappy-compressor compression/snappy-compressor) -(enc/defalias lzma2-compressor compression/lzma2-compressor) -(enc/defalias lz4-compressor compression/lz4-compressor) -(enc/defalias lz4hc-compressor compression/lz4hc-compressor) +(do + (enc/defalias compress compression/compress) + (enc/defalias decompress compression/decompress) + (enc/defalias snappy-compressor compression/snappy-compressor) + (enc/defalias lzma2-compressor compression/lzma2-compressor) + (enc/defalias lz4-compressor compression/lz4-compressor) + (enc/defalias lz4hc-compressor compression/lz4hc-compressor) -(enc/defalias encrypt encryption/encrypt) -(enc/defalias decrypt encryption/decrypt) -(enc/defalias aes128-encryptor encryption/aes128-encryptor) + (enc/defalias encrypt encryption/encrypt) + (enc/defalias decrypt encryption/decrypt) + (enc/defalias aes128-encryptor encryption/aes128-encryptor) -(enc/defalias freezable? utils/freezable?) + (enc/defalias freezable? utils/freezable?)) ;;;; Freezing (defprotocol Freezable - "Be careful about extending to interfaces, Ref. http://goo.gl/6gGRlU" + "Implementation detail. Be careful about extending to interfaces, + Ref. http://goo.gl/6gGRlU." (-freeze-to-out [this out])) -(defn small-count? [n] (<= (long n) 127 #_Byte/MAX_VALUE)) -(defmacro write-id [out id] `(.writeByte ~out ~id)) -(defmacro write-bytes [out ba & [small?]] - (let [wc (if small? 'writeByte 'writeInt) - out (with-meta out {:tag 'java.io.DataOutput}) - ba (with-meta ba {:tag 'bytes})] - `(let [out# ~out - ba# ~ba - size# (alength ba#)] - (. out# ~wc size#) - (.write out# ba# 0 size#)))) +(defmacro write-id [out id] `(.writeByte ~out ~id)) -(defmacro write-biginteger [out x] - (let [x (with-meta x {:tag 'java.math.BigInteger})] - `(write-bytes ~out (.toByteArray ~x)))) +(defn write-bytes [^DataOutput out ^bytes ba] + (let [len (alength ba)] + (.writeInt out len) + (.write out ba 0 len))) -(defmacro write-utf8 [out x & [small?]] - (let [x (with-meta x {:tag 'String})] - `(write-bytes ~out (.getBytes ~x "UTF-8") ~small?))) +(defn write-sm-bytes [^DataOutput out ^bytes ba] + (let [len (alength ba)] + (byte len) ; Safety check + (.writeByte out len) + (.write out ba 0 len))) + +(defn write-utf8 [^DataOutput out ^String s] (write-bytes out (.getBytes s "UTF-8"))) +(defn write-biginteger [^DataOutput out ^BigInteger n] (write-bytes out (.toByteArray n))) + +(defn byte-sized? [^long n] (<= n 127 #_Byte/MAX_VALUE)) +(defn write-ided-bytes [^DataOutput out id-sm id ^bytes ba] + (if (byte-sized? (alength ba)) + (do (write-id out id-sm) + (write-sm-bytes out ba)) + (do (write-id out id) + (write-bytes out ba)))) + +(defn write-ided-string [^DataOutput out ^String s] + (write-ided-bytes out id-sm-string id-string (.getBytes s "UTF-8"))) + +(defn write-ided-keyword [^DataOutput out kw] + (let [^String s (if-let [ns (namespace kw)] (str ns "/" (name kw)) (name kw))] + (write-ided-bytes out id-sm-keyword id-keyword (.getBytes s "UTF-8")))) + +(defn write-ided-long [^DataOutput out ^long n] + (cond + (and (<= n Byte/MAX_VALUE) (<= Byte/MIN_VALUE n)) + (do (write-id out id-byte-as-long) + (.writeByte out n)) + + (and (<= n Short/MAX_VALUE) (<= Short/MIN_VALUE n)) + (do (write-id out id-short-as-long) + (.writeShort out n)) + + (and (<= n Integer/MAX_VALUE) (<= Integer/MIN_VALUE n)) + (do (write-id out id-int-as-long) + (.writeInt out n)) + + :else + (do (write-id out id-long) + (.writeLong out n)))) (defn freeze-to-out! "Serializes arg (any Clojure data type) to a DataOutput" @@ -185,31 +215,50 @@ (-freeze-to-out m data-output)) (-freeze-to-out x data-output)) -(defmacro write-coll [out x & [small?]] - (let [wc (if small? 'writeByte 'writeInt)] - `(if (counted? ~'x) - (do - (. ~'out ~wc (count ~'x)) - (enc/run!* (fn [i#] (freeze-to-out! ~'out i#)) ~'x)) - (let [bas# (ByteArrayOutputStream. 64) - sout# (DataOutputStream. bas#) - cnt# (reduce (fn [^long cnt# i#] - (freeze-to-out! sout# i#) - (unchecked-inc cnt#)) - 0 ~'x) - ba# (.toByteArray bas#)] - (. ~'out ~wc cnt#) - (.write ~'out ba# 0 (alength ba#)))))) +(defn write-ided-coll [^DataOutput out ?id-sm id coll] + (if (counted? coll) + (let [cnt (count coll)] + (if (and ?id-sm (byte-sized? cnt)) + (do (write-id out ?id-sm) + (.writeByte out cnt)) + (do (write-id out id) + (.writeInt out cnt))) + (enc/run!* (fn [in] (freeze-to-out! out in)) coll)) -(defmacro write-kvs [out x & [small?]] - (let [wc (if small? 'writeByte 'writeInt)] - `(do - (. ~'out ~wc (count ~'x)) - (enc/run-kv! - (fn [k# v#] - (freeze-to-out! ~'out k#) - (freeze-to-out! ~'out v#)) - ~'x)))) + (let [bas (ByteArrayOutputStream. 64) + sout (DataOutputStream. bas) + cnt (reduce (fn [^long cnt in] + (freeze-to-out! sout in) + (unchecked-inc cnt)) + 0 coll) + ba (.toByteArray bas)] + + (if (and ?id-sm (byte-sized? cnt)) + (do (write-id out ?id-sm) + (.writeByte out cnt)) + (do (write-id out id) + (.writeInt out cnt))) + + (.write out ba 0 (alength ba))))) + +(defn write-ided-kvs [^DataOutput out ?id-sm id coll] + (let [cnt (count coll)] + (if (and ?id-sm (byte-sized? cnt)) + (do (write-id out ?id-sm) + (.writeByte out cnt)) + (do (write-id out id) + (.writeInt out cnt))) + (enc/run-kv! + (fn [k v] + (freeze-to-out! out k) + (freeze-to-out! out v)) + coll))) + +(defmacro ^:private freezer* [type & body] + `(extend-type ~type + Freezable + (~'-freeze-to-out [~'x ~(with-meta 'out {:tag 'DataOutput})] + ~@body))) (defmacro ^:private freezer [type id & body] `(extend-type ~type @@ -218,122 +267,54 @@ (write-id ~'out ~id) ~@body))) -(defmacro ^:private freezer-coll [type id & [id-sm]] - (if-not id-sm - `(freezer ~type ~id (write-coll ~'out ~'x)) - `(extend-type ~type - Freezable - (~'-freeze-to-out [~'x ~(with-meta 'out {:tag 'DataOutput})] - (if (small-count? (count ~'x)) - (do - (write-id ~'out ~id-sm) - (write-coll ~'out ~'x :small)) - (do - (write-id ~'out ~id) - (write-coll ~'out ~'x))))))) +(freezer nil id-nil) +(freezer (Class/forName "[B") id-bytes (write-bytes out x)) +(freezer Boolean id-boolean (.writeBoolean out x)) +(freezer Character id-char (.writeChar out (int x))) +(freezer* String (write-ided-string out x)) +(freezer* Keyword (write-ided-keyword out x)) -(defmacro ^:private freezer-kvs [type id & [id-sm]] - (if-not id-sm - `(freezer ~type ~id (write-kvs ~'out ~'x)) - `(extend-type ~type - Freezable - (~'-freeze-to-out [~'x ~(with-meta 'out {:tag 'DataOutput})] - (if (small-count? (count ~'x)) - (do - (write-id ~'out ~id-sm) - (write-kvs ~'out ~'x :small)) - (do - (write-id ~'out ~id) - (write-kvs ~'out ~'x))))))) +(freezer* PersistentQueue (write-ided-coll out nil id-queue x)) +(freezer* PersistentTreeSet (write-ided-coll out nil id-sorted-set x)) +(freezer* PersistentTreeMap (write-ided-kvs out nil id-sorted-map x)) +(freezer* APersistentMap (write-ided-kvs out id-sm-map id-map x)) +(freezer* APersistentVector (write-ided-coll out id-sm-vector id-vector x)) +(freezer* APersistentSet (write-ided-coll out id-sm-set id-set x)) -(freezer (Class/forName "[B") id-bytes (write-bytes out ^bytes x)) -(freezer nil id-nil) -(freezer Boolean id-boolean (.writeBoolean out x)) -(freezer Character id-char (.writeChar out (int x))) - -(extend-type String - Freezable - (-freeze-to-out [x ^DataOutput out] - (let [ba (.getBytes x "UTF-8")] - (if (small-count? (alength ^bytes ba)) - (do (write-id out id-sm-string) - (write-bytes out ba :small)) - (do (write-id out id-string) - (write-bytes out ba)))))) - -(extend-type Keyword - Freezable - (-freeze-to-out [x ^DataOutput out] - (let [s (if-let [ns (namespace x)] (str ns "/" (name x)) (name x)) - ba (.getBytes s "UTF-8")] - (if (small-count? Byte/MAX_VALUE) - (do (write-id out id-sm-keyword) - (write-bytes out ba :small)) - (do (write-id out id-keyword) - (write-bytes out ba)))))) - -(freezer-coll PersistentQueue id-queue) -(freezer-coll PersistentTreeSet id-sorted-set) -(freezer-kvs PersistentTreeMap id-sorted-map) - -(freezer-kvs APersistentMap id-map id-sm-map) -(freezer-coll APersistentVector id-vector id-sm-vector) -(freezer-coll APersistentSet id-set id-sm-set) -(freezer-coll PersistentList id-list) ; No APersistentList -(freezer-coll (type '()) id-list) +;; No APersistentList: +(freezer* PersistentList (write-ided-coll out nil id-list x)) +(freezer* (type '()) (write-ided-coll out nil id-list x)) ;; Nb low-level interface!! Acts as fallback for seqs that don't have a ;; concrete implementation. Will conflict with any other coll interfaces! -(freezer-coll ISeq id-seq) +(freezer* ISeq (write-ided-coll out nil id-seq x)) -(freezer IRecord id-record - (write-utf8 out (.getName (class x))) ; Reflect - (freeze-to-out! out (into {} x))) +(freezer IRecord id-record + (write-utf8 out (.getName (class x))) ; Reflect + (freeze-to-out! out (into {} x))) -(freezer Byte id-byte (.writeByte out x)) -(freezer Short id-short (.writeShort out x)) -(freezer Integer id-integer (.writeInt out x)) -;;(freezer Long id-long (.writeLong out x)) -(extend-type Long - Freezable - (-freeze-to-out [x ^DataOutput out] - (let [^long x x] - (cond - (and (<= x #_Byte/MAX_VALUE 127) - (<= #_Byte/MIN_VALUE -128 x)) - (do (write-id out id-byte-as-long) - (.writeByte out x)) +(freezer Byte id-byte (.writeByte out x)) +(freezer Short id-short (.writeShort out x)) +(freezer Integer id-integer (.writeInt out x)) +(freezer* Long (write-ided-long out x)) - (and (<= x #_Short/MAX_VALUE 32767) - (<= #_Short/MIN_VALUE -32768 x)) - (do (write-id out id-short-as-long) - (.writeShort out x)) +(freezer BigInt id-bigint (write-biginteger out (.toBigInteger x))) +(freezer BigInteger id-biginteger (write-biginteger out x)) - (and (<= x #_Integer/MAX_VALUE 2147483647) - (<= #_Integer/MIN_VALUE -2147483648 x)) - (do (write-id out id-int-as-long) - (.writeInt out x)) - - :else (do (write-id out id-long) - (.writeLong out x)))))) - -(freezer BigInt id-bigint (write-biginteger out (.toBigInteger x))) -(freezer BigInteger id-biginteger (write-biginteger out x)) - -(freezer Float id-float (.writeFloat out x)) -(freezer Double id-double (.writeDouble out x)) -(freezer BigDecimal id-bigdec - (write-biginteger out (.unscaledValue x)) - (.writeInt out (.scale x))) +(freezer Float id-float (.writeFloat out x)) +(freezer Double id-double (.writeDouble out x)) +(freezer BigDecimal id-bigdec + (write-biginteger out (.unscaledValue x)) + (.writeInt out (.scale x))) (freezer Ratio id-ratio - (write-biginteger out (.numerator x)) - (write-biginteger out (.denominator x))) + (write-biginteger out (.numerator x)) + (write-biginteger out (.denominator x))) (freezer Date id-date (.writeLong out (.getTime x))) (freezer UUID id-uuid - (.writeLong out (.getMostSignificantBits x)) - (.writeLong out (.getLeastSignificantBits x))) + (.writeLong out (.getMostSignificantBits x)) + (.writeLong out (.getLeastSignificantBits x))) (enc/defonce* ^:dynamic *final-freeze-fallback* nil) (defn freeze-fallback-as-str [out x] @@ -351,16 +332,14 @@ (-freeze-to-out [x ^DataOutput out] (cond (utils/serializable? x) ; Fallback #1: Java's Serializable interface - (do (when-debug-mode - (println (format "DEBUG - Serializable fallback: %s" (type x)))) - (write-id out id-serializable) + (do (when-debug-mode (println (format "DEBUG - Serializable fallback: %s" (type x)))) + (write-id out id-serializable) (write-utf8 out (.getName (class x))) ; Reflect (.writeObject (ObjectOutputStream. out) x)) (utils/readable? x) ; Fallback #2: Clojure's Reader - (do (when-debug-mode - (println (format "DEBUG - Reader fallback: %s" (type x)))) - (write-id out id-reader) + (do (when-debug-mode (println (format "DEBUG - Reader fallback: %s" (type x)))) + (write-id out id-reader) (write-utf8 out (enc/pr-edn x))) :else ; Fallback #3: *final-freeze-fallback* @@ -563,11 +542,11 @@ :throwable e :nippy/unthawable {:class-name class-name :content content}}))) - id-bytes (read-bytes in) id-nil nil + id-bytes (read-bytes in) id-boolean (.readBoolean in) + id-char (.readChar in) - id-char (.readChar in) id-string (read-utf8 in) id-keyword (keyword (read-utf8 in)) @@ -604,15 +583,12 @@ id-int-as-long (long (.readInt in)) id-bigint (bigint (read-biginteger in)) - id-biginteger (read-biginteger in) + id-biginteger (read-biginteger in) id-float (.readFloat in) id-double (.readDouble in) id-bigdec (BigDecimal. (read-biginteger in) (.readInt in)) - ;; id-ratio (/ (bigint (read-biginteger in)) - ;; (bigint (read-biginteger in))) - id-ratio (clojure.lang.Ratio. (read-biginteger in) (read-biginteger in)) @@ -624,10 +600,11 @@ id-sorted-map-depr1 (read-kvs-depr1 in (sorted-map)) id-map-depr2 (read-kvs-depr1 in {}) id-reader-depr1 (enc/read-edn (.readUTF in)) - id-string-depr1 (.readUTF in) - id-map-depr1 (apply hash-map (enc/repeatedly-into [] (* 2 (.readInt in)) - (fn [] (thaw-from-in! in)))) + id-string-depr1 (.readUTF in) id-keyword-depr1 (keyword (.readUTF in)) + id-map-depr1 (apply hash-map + (enc/repeatedly-into [] (* 2 (.readInt in)) + (fn [] (thaw-from-in! in)))) id-prefixed-custom ; Prefixed custom type (let [hash-id (.readShort in)] @@ -857,64 +834,63 @@ (defrecord StressRecord [data]) (def stress-data "Reference data used for tests & benchmarks" - (let [] - {:bytes (byte-array [(byte 1) (byte 2) (byte 3)]) - :nil nil - :boolean true + {:bytes (byte-array [(byte 1) (byte 2) (byte 3)]) + :nil nil + :boolean true - :char-utf8 \ಬ - :string-utf8 "ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸ" - :string-long (apply str (range 1000)) - :keyword :keyword - :keyword-ns ::keyword + :char-utf8 \ಬ + :string-utf8 "ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸ" + :string-long (apply str (range 1000)) + :keyword :keyword + :keyword-ns ::keyword - ;;; Try reflect real-world data: - :lotsa-small-numbers (vec (range 200)) - :lotsa-small-keywords (->> (java.util.Locale/getISOLanguages) - (mapv keyword)) - :lotsa-small-strings (->> (java.util.Locale/getISOCountries) - (mapv #(.getDisplayCountry - (java.util.Locale. "en" %)))) + ;;; Try reflect real-world data: + :lotsa-small-numbers (vec (range 200)) + :lotsa-small-keywords (->> (java.util.Locale/getISOLanguages) + (mapv keyword)) + :lotsa-small-strings (->> (java.util.Locale/getISOCountries) + (mapv #(.getDisplayCountry + (java.util.Locale. "en" %)))) - :queue (-> (PersistentQueue/EMPTY) (conj :a :b :c :d :e :f :g)) - :queue-empty (PersistentQueue/EMPTY) - :sorted-set (sorted-set 1 2 3 4 5) - :sorted-map (sorted-map :b 2 :a 1 :d 4 :c 3) + :queue (-> (PersistentQueue/EMPTY) (conj :a :b :c :d :e :f :g)) + :queue-empty (PersistentQueue/EMPTY) + :sorted-set (sorted-set 1 2 3 4 5) + :sorted-map (sorted-map :b 2 :a 1 :d 4 :c 3) - :list (list 1 2 3 4 5 (list 6 7 8 (list 9 10))) - :list-quoted '(1 2 3 4 5 (6 7 8 (9 10))) - :list-empty (list) - :vector [1 2 3 4 5 [6 7 8 [9 10]]] - :vector-empty [] - :map {:a 1 :b 2 :c 3 :d {:e 4 :f {:g 5 :h 6 :i 7}}} - :map-empty {} - :set #{1 2 3 4 5 #{6 7 8 #{9 10}}} - :set-empty #{} - :meta (with-meta {:a :A} {:metakey :metaval}) + :list (list 1 2 3 4 5 (list 6 7 8 (list 9 10))) + :list-quoted '(1 2 3 4 5 (6 7 8 (9 10))) + :list-empty (list) + :vector [1 2 3 4 5 [6 7 8 [9 10]]] + :vector-empty [] + :map {:a 1 :b 2 :c 3 :d {:e 4 :f {:g 5 :h 6 :i 7}}} + :map-empty {} + :set #{1 2 3 4 5 #{6 7 8 #{9 10}}} + :set-empty #{} + :meta (with-meta {:a :A} {:metakey :metaval}) - :lazy-seq (repeatedly 1000 rand) - :lazy-seq-empty (map identity '()) + :lazy-seq (repeatedly 1000 rand) + :lazy-seq-empty (map identity '()) - :byte (byte 16) - :short (short 42) - :integer (int 3) - :long (long 3) - :bigint (bigint 31415926535897932384626433832795) + :byte (byte 16) + :short (short 42) + :integer (int 3) + :long (long 3) + :bigint (bigint 31415926535897932384626433832795) - :float (float 3.14) - :double (double 3.14) - :bigdec (bigdec 3.1415926535897932384626433832795) + :float (float 3.14) + :double (double 3.14) + :bigdec (bigdec 3.1415926535897932384626433832795) - :ratio 22/7 - :uuid (java.util.UUID/randomUUID) - :date (java.util.Date.) + :ratio 22/7 + :uuid (java.util.UUID/randomUUID) + :date (java.util.Date.) - :stress-record (->StressRecord "data") + :stress-record (->StressRecord "data") - ;; Serializable - :throwable (Throwable. "Yolo") - :exception (try (/ 1 0) (catch Exception e e)) - :ex-info (ex-info "ExInfo" {:data "data"})})) + ;; Serializable + :throwable (Throwable. "Yolo") + :exception (try (/ 1 0) (catch Exception e e)) + :ex-info (ex-info "ExInfo" {:data "data"})}) (def stress-data-comparable "Reference data with stuff removed that breaks roundtrip equality" diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 010b446..426d773 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -65,9 +65,15 @@ (set! *unchecked-math* false) ;; (bench {:reader? true :lzma2? true :fressian? true :laps 3}) ;; (bench {:laps 4}) - ;; (bench {:laps 1 :lzma2? true}) + ;; (bench {:laps 2 :lzma2? true}) ;; (bench {:laps 1}) + ;;; 2015 Sep 29, after read/write API refactor + {:lzma2 {:round 54319, :freeze 36084, :thaw 18235, :size 11264}} + {:default {:round 5597, :freeze 3592, :thaw 2005, :size 16109}} + {:fast {:round 4889, :freeze 2979, :thaw 1910, :size 16972}} + {:encrypted {:round 6228, :freeze 4031, :thaw 2197, :size 16132}} + ;;; 2015 Sep 28, small collection optimizations {:lzma2 {:round 56307, :freeze 36475, :thaw 19832, :size 11244}} {:encrypted {:round 6062, :freeze 3802, :thaw 2260, :size 16148}} diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index a05e17a..964c99e 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -17,7 +17,8 @@ (expect (do (println (str "Clojure version: " *clojure-version*)) true)) (expect test-data ((comp thaw freeze) test-data)) -(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor}) +(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor + :encryptor nil}) #(freeze % {:skip-header? true})) test-data)) (expect test-data ((comp #(thaw % {:password [:salted "p"]}) From 2ebd8ce2ac84fd6fd1308d341c819222a5443702 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 19:59:53 +0700 Subject: [PATCH 14/33] Fix id typing --- src/taoensso/nippy.clj | 106 ++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 464eded..d02293b 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -64,74 +64,74 @@ (do ;; ** Negative ids reserved for user-defined types ** ;; - (def ^:const id-reserved (int 0)) - ;; 1 ; Deprecated - (def ^:const id-bytes (int 2)) - (def ^:const id-nil (int 3)) - (def ^:const id-boolean (int 4)) - (def ^:const id-reader (int 5)) ; Fallback #2 - (def ^:const id-serializable (int 6)) ; Fallback #1 + (def ^:const id-reserved (byte 0)) + ;; 1 ; Deprecated + (def ^:const id-bytes (byte 2)) + (def ^:const id-nil (byte 3)) + (def ^:const id-boolean (byte 4)) + (def ^:const id-reader (byte 5)) ; Fallback #2 + (def ^:const id-serializable (byte 6)) ; Fallback #1 - (def ^:const id-char (int 10)) - ;; 11 ; Deprecated - ;; 12 ; Deprecated - (def ^:const id-string (int 13)) - (def ^:const id-keyword (int 14)) + (def ^:const id-char (byte 10)) + ;; 11 ; Deprecated + ; ; 12 ; Deprecated + (def ^:const id-string (byte 13)) + (def ^:const id-keyword (byte 14)) - (def ^:const id-list (int 20)) - (def ^:const id-vector (int 21)) - ;; 22 ; Deprecated - (def ^:const id-set (int 23)) - (def ^:const id-seq (int 24)) - (def ^:const id-meta (int 25)) - (def ^:const id-queue (int 26)) - ;; 27 ; Deprecated - (def ^:const id-sorted-set (int 28)) - ;; 29 ; Deprecated - (def ^:const id-map (int 30)) - (def ^:const id-sorted-map (int 31)) + (def ^:const id-list (byte 20)) + (def ^:const id-vector (byte 21)) + ;; 22 ; Deprecated + (def ^:const id-set (byte 23)) + (def ^:const id-seq (byte 24)) + (def ^:const id-meta (byte 25)) + (def ^:const id-queue (byte 26)) + ;; 27 ; Deprecated + (def ^:const id-sorted-set (byte 28)) + ;; 29 ; Deprecated + (def ^:const id-map (byte 30)) + (def ^:const id-sorted-map (byte 31)) - (def ^:const id-byte (int 40)) - (def ^:const id-short (int 41)) - (def ^:const id-integer (int 42)) - (def ^:const id-long (int 43)) - (def ^:const id-bigint (int 44)) - (def ^:const id-biginteger (int 45)) + (def ^:const id-byte (byte 40)) + (def ^:const id-short (byte 41)) + (def ^:const id-integer (byte 42)) + (def ^:const id-long (byte 43)) + (def ^:const id-bigint (byte 44)) + (def ^:const id-biginteger (byte 45)) - (def ^:const id-float (int 60)) - (def ^:const id-double (int 61)) - (def ^:const id-bigdec (int 62)) + (def ^:const id-float (byte 60)) + (def ^:const id-double (byte 61)) + (def ^:const id-bigdec (byte 62)) - (def ^:const id-ratio (int 70)) + (def ^:const id-ratio (byte 70)) - (def ^:const id-record (int 80)) - ;; (def ^:const id-type (int 81)) ; TODO? - (def ^:const id-prefixed-custom (int 82)) + (def ^:const id-record (byte 80)) + ;; (def ^:const id-type (byte 81)) ; TODO? + (def ^:const id-prefixed-custom (byte 82)) - (def ^:const id-date (int 90)) - (def ^:const id-uuid (int 91)) + (def ^:const id-date (byte 90)) + (def ^:const id-uuid (byte 91)) ;;; Optimized, common-case types (v2.6+) - (def ^:const id-byte-as-long (int 100)) ; 1 vs 8 bytes - (def ^:const id-short-as-long (int 101)) ; 2 vs 8 bytes - (def ^:const id-int-as-long (int 102)) ; 4 vs 8 bytes + (def ^:const id-byte-as-long (byte 100)) ; 1 vs 8 bytes + (def ^:const id-short-as-long (byte 101)) ; 2 vs 8 bytes + (def ^:const id-int-as-long (byte 102)) ; 4 vs 8 bytes ;; - (def ^:const id-sm-string (int 105)) ; 1 vs 4 byte length prefix - (def ^:const id-sm-keyword (int 106)) ; '' + (def ^:const id-sm-string (byte 105)) ; 1 vs 4 byte length prefix + (def ^:const id-sm-keyword (byte 106)) ; '' ;; - (def ^:const id-sm-vector (int 110)) ; '' - (def ^:const id-sm-set (int 111)) ; '' - (def ^:const id-sm-map (int 112)) ; '' + (def ^:const id-sm-vector (byte 110)) ; '' + (def ^:const id-sm-set (byte 111)) ; '' + (def ^:const id-sm-map (byte 112)) ; '' ;; ;; TODO Additional optimizations (types) for 2-vecs and 3-vecs? ;;; DEPRECATED (old types will be supported only for thawing) - (def ^:const id-reader-depr1 (int 1)) ; v0.9.2+ for +64k support - (def ^:const id-string-depr1 (int 11)) ; v0.9.2+ for +64k support - (def ^:const id-map-depr1 (int 22)) ; v0.9.0+ for more efficient thaw - (def ^:const id-keyword-depr1 (int 12)) ; v2.0.0-alpha5+ for str consistecy - (def ^:const id-map-depr2 (int 27)) ; v2.11+ for count/2 - (def ^:const id-sorted-map-depr1 (int 29)) ; v2.11+ for count/2 + (def ^:const id-reader-depr1 (byte 1)) ; v0.9.2+ for +64k support + (def ^:const id-string-depr1 (byte 11)) ; v0.9.2+ for +64k support + (def ^:const id-map-depr1 (byte 22)) ; v0.9.0+ for more efficient thaw + (def ^:const id-keyword-depr1 (byte 12)) ; v2.0.0-alpha5+ for str consistecy + (def ^:const id-map-depr2 (byte 27)) ; v2.11+ for count/2 + (def ^:const id-sorted-map-depr1 (byte 29)) ; v2.11+ for count/2 ) ;;;; Ns imports (mostly for convenience of lib consumers) From d61fb06f3ba02cfbfbf177bc366832dbf23bf0b5 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 20:05:41 +0700 Subject: [PATCH 15/33] Primitive ided-long checks --- src/taoensso/nippy.clj | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index d02293b..f0f9d77 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -190,15 +190,18 @@ (defn write-ided-long [^DataOutput out ^long n] (cond - (and (<= n Byte/MAX_VALUE) (<= Byte/MIN_VALUE n)) + (and (<= n 127 #_Byte/MAX_VALUE) + (<= -128 #_Byte/MIN_VALUE n)) (do (write-id out id-byte-as-long) (.writeByte out n)) - (and (<= n Short/MAX_VALUE) (<= Short/MIN_VALUE n)) + (and (<= n 32767 #_Short/MAX_VALUE) + (<= -32768 #_Short/MIN_VALUE n)) (do (write-id out id-short-as-long) (.writeShort out n)) - (and (<= n Integer/MAX_VALUE) (<= Integer/MIN_VALUE n)) + (and (<= n 2147483647 #_Integer/MAX_VALUE) + (<= -2147483648 #_Integer/MIN_VALUE n)) (do (write-id out id-int-as-long) (.writeInt out n)) From 7072f73952c90b1752a7d21db4a8beea698bdfac Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 20:10:09 +0700 Subject: [PATCH 16/33] Misc hk --- project.clj | 2 +- src/taoensso/nippy.clj | 87 ++++++++++++++++--------------- src/taoensso/nippy/benchmarks.clj | 10 ++-- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/project.clj b/project.clj index bb35c85..2fc5db4 100644 --- a/project.clj +++ b/project.clj @@ -9,7 +9,7 @@ :min-lein-version "2.3.3" :global-vars {*warn-on-reflection* true *assert* true - *unchecked-math* :warn-on-boxed} + *unchecked-math* false} :dependencies [[org.clojure/clojure "1.5.1"] diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index f0f9d77..1869215 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -57,7 +57,7 @@ (byte 12) {:version 1 :compressor-id :lzma2 :encryptor-id :aes128-sha512} (byte 13) {:version 1 :compressor-id :lzma2 :encryptor-id :else}}) -(defmacro when-debug-mode [& body] (when #_true false `(do ~@body))) +(defmacro when-debug [& body] (when #_true false `(do ~@body))) ;;;; Data type IDs @@ -79,7 +79,7 @@ (def ^:const id-keyword (byte 14)) (def ^:const id-list (byte 20)) - (def ^:const id-vector (byte 21)) + (def ^:const id-vec (byte 21)) ;; 22 ; Deprecated (def ^:const id-set (byte 23)) (def ^:const id-seq (byte 24)) @@ -119,7 +119,7 @@ (def ^:const id-sm-string (byte 105)) ; 1 vs 4 byte length prefix (def ^:const id-sm-keyword (byte 106)) ; '' ;; - (def ^:const id-sm-vector (byte 110)) ; '' + (def ^:const id-sm-vec (byte 110)) ; '' (def ^:const id-sm-set (byte 111)) ; '' (def ^:const id-sm-map (byte 112)) ; '' ;; @@ -158,7 +158,6 @@ (-freeze-to-out [this out])) (defmacro write-id [out id] `(.writeByte ~out ~id)) - (defn write-bytes [^DataOutput out ^bytes ba] (let [len (alength ba)] (.writeInt out len) @@ -170,10 +169,13 @@ (.writeByte out len) (.write out ba 0 len))) -(defn write-utf8 [^DataOutput out ^String s] (write-bytes out (.getBytes s "UTF-8"))) -(defn write-biginteger [^DataOutput out ^BigInteger n] (write-bytes out (.toByteArray n))) +(defn write-biginteger [out ^BigInteger n] (write-bytes out (.toByteArray n))) +(defn write-utf8 [out ^String s] (write-bytes out (.getBytes s "UTF-8"))) +(defn write-sm-utf8 [out ^String s] (write-sm-bytes out (.getBytes s "UTF-8"))) + +(defn byte-sized? [^long n] (<= n 127 #_Byte/MAX_VALUE)) +(defn short-sized? [^long n] (<= n 32767 #_Short/MAX_VALUE)) -(defn byte-sized? [^long n] (<= n 127 #_Byte/MAX_VALUE)) (defn write-ided-bytes [^DataOutput out id-sm id ^bytes ba] (if (byte-sized? (alength ba)) (do (write-id out id-sm) @@ -181,27 +183,27 @@ (do (write-id out id) (write-bytes out ba)))) -(defn write-ided-string [^DataOutput out ^String s] +(defn write-ided-string [out ^String s] (write-ided-bytes out id-sm-string id-string (.getBytes s "UTF-8"))) -(defn write-ided-keyword [^DataOutput out kw] +(defn write-ided-keyword [out kw] (let [^String s (if-let [ns (namespace kw)] (str ns "/" (name kw)) (name kw))] (write-ided-bytes out id-sm-keyword id-keyword (.getBytes s "UTF-8")))) (defn write-ided-long [^DataOutput out ^long n] (cond - (and (<= n 127 #_Byte/MAX_VALUE) - (<= -128 #_Byte/MIN_VALUE n)) + (and (<= n 127 #_Byte/MAX_VALUE) + (>= n -128 #_Byte/MIN_VALUE)) (do (write-id out id-byte-as-long) (.writeByte out n)) - (and (<= n 32767 #_Short/MAX_VALUE) - (<= -32768 #_Short/MIN_VALUE n)) + (and (<= n 32767 #_Short/MAX_VALUE) + (>= n -32768 #_Short/MIN_VALUE)) (do (write-id out id-short-as-long) (.writeShort out n)) (and (<= n 2147483647 #_Integer/MAX_VALUE) - (<= -2147483648 #_Integer/MIN_VALUE n)) + (>= -2147483648 #_Integer/MIN_VALUE)) (do (write-id out id-int-as-long) (.writeInt out n)) @@ -257,6 +259,10 @@ (freeze-to-out! out v)) coll))) +(defn write-ided-vec [out v] (write-ided-coll out id-sm-vec id-vec v)) +(defn write-ided-set [out s] (write-ided-coll out id-sm-set id-set s)) +(defn write-ided-map [out m] (write-ided-kvs out id-sm-map id-map m)) + (defmacro ^:private freezer* [type & body] `(extend-type ~type Freezable @@ -277,24 +283,24 @@ (freezer* String (write-ided-string out x)) (freezer* Keyword (write-ided-keyword out x)) -(freezer* PersistentQueue (write-ided-coll out nil id-queue x)) -(freezer* PersistentTreeSet (write-ided-coll out nil id-sorted-set x)) -(freezer* PersistentTreeMap (write-ided-kvs out nil id-sorted-map x)) -(freezer* APersistentMap (write-ided-kvs out id-sm-map id-map x)) -(freezer* APersistentVector (write-ided-coll out id-sm-vector id-vector x)) -(freezer* APersistentSet (write-ided-coll out id-sm-set id-set x)) +(freezer* PersistentQueue (write-ided-coll out nil id-queue x)) +(freezer* PersistentTreeSet (write-ided-coll out nil id-sorted-set x)) +(freezer* PersistentTreeMap (write-ided-kvs out nil id-sorted-map x)) +(freezer* APersistentMap (write-ided-kvs out id-sm-map id-map x)) +(freezer* APersistentVector (write-ided-coll out id-sm-vec id-vec x)) +(freezer* APersistentSet (write-ided-coll out id-sm-set id-set x)) ;; No APersistentList: -(freezer* PersistentList (write-ided-coll out nil id-list x)) -(freezer* (type '()) (write-ided-coll out nil id-list x)) +(freezer* PersistentList (write-ided-coll out nil id-list x)) +(freezer* (type '()) (write-ided-coll out nil id-list x)) ;; Nb low-level interface!! Acts as fallback for seqs that don't have a ;; concrete implementation. Will conflict with any other coll interfaces! -(freezer* ISeq (write-ided-coll out nil id-seq x)) +(freezer* ISeq (write-ided-coll out nil id-seq x)) (freezer IRecord id-record (write-utf8 out (.getName (class x))) ; Reflect - (freeze-to-out! out (into {} x))) + (-freeze-to-out (into {} x) out)) (freezer Byte id-byte (.writeByte out x)) (freezer Short id-short (.writeShort out x)) @@ -335,13 +341,13 @@ (-freeze-to-out [x ^DataOutput out] (cond (utils/serializable? x) ; Fallback #1: Java's Serializable interface - (do (when-debug-mode (println (format "DEBUG - Serializable fallback: %s" (type x)))) + (do (when-debug (println (format "DEBUG - Serializable fallback: %s" (type x)))) (write-id out id-serializable) (write-utf8 out (.getName (class x))) ; Reflect (.writeObject (ObjectOutputStream. out) x)) (utils/readable? x) ; Fallback #2: Clojure's Reader - (do (when-debug-mode (println (format "DEBUG - Reader fallback: %s" (type x)))) + (do (when-debug (println (format "DEBUG - Reader fallback: %s" (type x)))) (write-id out id-reader) (write-utf8 out (enc/pr-edn x))) @@ -396,8 +402,7 @@ (^bytes [x] (freeze x nil)) (^bytes [x {:keys [compressor encryptor password skip-header?] :or {compressor :auto - encryptor aes128-encryptor} - :as opts}] + encryptor aes128-encryptor}}] (let [encryptor (when password encryptor) zero-copy-mode? (and (nil? compressor) (nil? encryptor)) baos (ByteArrayOutputStream. 64) @@ -443,15 +448,15 @@ (declare thaw-from-in!) (defn read-bytes ^bytes [^DataInput in] - (let [size (.readInt in) - ba (byte-array size)] - (.readFully in ba 0 size) + (let [len (.readInt in) + ba (byte-array len)] + (.readFully in ba 0 len) ba)) (defn read-sm-bytes ^bytes [^DataInput in] - (let [size (.readByte in) - ba (byte-array size)] - (.readFully in ba 0 size) + (let [len (.readByte in) + ba (byte-array len)] + (.readFully in ba 0 len) ba)) (defn read-biginteger ^BigInteger [^DataInput in] (BigInteger. (read-bytes in))) @@ -502,10 +507,8 @@ [^DataInput data-input] (let [in data-input type-id (.readByte in)] + (when-debug (println (format "DEBUG - thawing type-id: %s" type-id))) (try - (when-debug-mode - (println (format "DEBUG - thawing type-id: %s" type-id))) - (enc/case-eval type-id id-reader @@ -561,8 +564,8 @@ id-sorted-set (read-coll in (sorted-set)) id-sorted-map (read-kvs in (sorted-map)) - id-vector (read-coll in []) - id-sm-vector (read-sm-coll in []) + id-vec (read-coll in []) + id-sm-vec (read-sm-coll in []) id-set (read-coll in #{}) id-sm-set (read-sm-coll in #{}) id-map (read-kvs in {}) @@ -926,9 +929,9 @@ (catch Exception _ false)) :unwrapped-ba unwrapped-ba :data-ba data-ba - :unwrapped-size (alength ^bytes unwrapped-ba) - :ba-size (alength ^bytes ba) - :data-size (alength ^bytes data-ba)}))) + :unwrapped-len (alength ^bytes unwrapped-ba) + :ba-len (alength ^bytes ba) + :data-len (alength ^bytes data-ba)}))) (comment (inspect-ba (freeze "hello")) (seq (:data-ba (inspect-ba (freeze "hello"))))) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 426d773..30860c9 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -66,13 +66,13 @@ ;; (bench {:reader? true :lzma2? true :fressian? true :laps 3}) ;; (bench {:laps 4}) ;; (bench {:laps 2 :lzma2? true}) - ;; (bench {:laps 1}) + ;; (bench {:laps 2}) ;;; 2015 Sep 29, after read/write API refactor - {:lzma2 {:round 54319, :freeze 36084, :thaw 18235, :size 11264}} - {:default {:round 5597, :freeze 3592, :thaw 2005, :size 16109}} - {:fast {:round 4889, :freeze 2979, :thaw 1910, :size 16972}} - {:encrypted {:round 6228, :freeze 4031, :thaw 2197, :size 16132}} + {:lzma2 {:round 51640, :freeze 33699, :thaw 17941, :size 11240}} + {:encrypted {:round 5922, :freeze 3734, :thaw 2188, :size 16132}} + {:default {:round 5588, :freeze 3658, :thaw 1930, :size 16113}} + {:fast {:round 4533, :freeze 2688, :thaw 1845, :size 16972}} ;;; 2015 Sep 28, small collection optimizations {:lzma2 {:round 56307, :freeze 36475, :thaw 19832, :size 11244}} From 7faaf48ee7727006b0b7f66cbf0e197e99acfd6b Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 23:02:24 +0700 Subject: [PATCH 17/33] Deprecate `Compressable-LZMA2` (was anyway marked as experimental) --- src/taoensso/nippy.clj | 58 +++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 1869215..00bd4c8 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -807,35 +807,6 @@ (extend-thaw 1 [in] (->MyType (.readUTF in))) (thaw (freeze (->MyType "Joe")))) -;;; Some useful custom types - EXPERIMENTAL - -;; Mostly deprecated by :auto compressor selection -(defrecord Compressable-LZMA2 [value]) ; Why was this `LZMA2` instead of `lzma2`? -(extend-freeze Compressable-LZMA2 128 [x out] - (let [ba (freeze (:value x) {:skip-header? true :compressor nil}) - ba-len (alength ba) - compress? (> ba-len 1024)] - (.writeBoolean out compress?) - (if compress? - (write-bytes out (compress lzma2-compressor ba)) - (write-bytes out ba)))) - -(extend-thaw 128 [in] - (let [compressed? (.readBoolean in) - ba (read-bytes in)] - (thaw ba {:compressor (when compressed? lzma2-compressor) - :encryptor nil}))) - -(comment - (->> (apply str (repeatedly 1000 rand)) - (->Compressable-LZMA2) - (freeze) - (thaw)) - (count (->> (apply str (repeatedly 1000 rand)) (freeze))) - (count (->> (apply str (repeatedly 1000 rand)) - (->Compressable-LZMA2) - (freeze)))) - ;;;; Stress data (defrecord StressRecord [data]) @@ -935,3 +906,32 @@ (comment (inspect-ba (freeze "hello")) (seq (:data-ba (inspect-ba (freeze "hello"))))) + +;;;; Deprecated + +;; Deprecated by :auto compressor selection +(defrecord Compressable-LZMA2 [value]) ; Why was this `LZMA2` instead of `lzma2`? +(extend-freeze Compressable-LZMA2 128 [x out] + (let [ba (freeze (:value x) {:skip-header? true :compressor nil}) + ba-len (alength ba) + compress? (> ba-len 1024)] + (.writeBoolean out compress?) + (if compress? + (write-bytes out (compress lzma2-compressor ba)) + (write-bytes out ba)))) + +(extend-thaw 128 [in] + (let [compressed? (.readBoolean in) + ba (read-bytes in)] + (thaw ba {:compressor (when compressed? lzma2-compressor) + :encryptor nil}))) + +(comment + (->> (apply str (repeatedly 1000 rand)) + (->Compressable-LZMA2) + (freeze) + (thaw)) + (count (->> (apply str (repeatedly 1000 rand)) (freeze))) + (count (->> (apply str (repeatedly 1000 rand)) + (->Compressable-LZMA2) + (freeze)))) From c5901730ea20d52711d38d887aabcedc325cf1a0 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 23:06:33 +0700 Subject: [PATCH 18/33] Update ba inspector --- src/taoensso/nippy.clj | 44 ++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 00bd4c8..1991329 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -882,30 +882,32 @@ ;;;; Tools (defn inspect-ba "Alpha - subject to change" - [ba & [thaw-opts]] - (if-not (enc/bytes? ba) :not-ba - (let [[first2bytes nextbytes] (enc/ba-split ba 2) - known-wrapper - (cond - (enc/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin - (enc/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj) + ([ba ] (inspect-ba ba nil)) + ([ba thaw-opts] + (when (enc/bytes? ba) + (let [[first2bytes nextbytes] (enc/ba-split ba 2) + ?known-wrapper + (cond + (enc/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin + (enc/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj) - unwrapped-ba (if known-wrapper nextbytes ba) - [data-ba nippy-header] (or (try-parse-header unwrapped-ba) - [unwrapped-ba :no-header])] + unwrapped-ba (if ?known-wrapper nextbytes ba) + [data-ba ?nippy-header] (or (try-parse-header unwrapped-ba) + [unwrapped-ba :no-header])] - {:known-wrapper known-wrapper - :nippy-v2-header nippy-header ; Nippy v1.x didn't have a header - :thawable? (try (thaw unwrapped-ba thaw-opts) true - (catch Exception _ false)) - :unwrapped-ba unwrapped-ba - :data-ba data-ba - :unwrapped-len (alength ^bytes unwrapped-ba) - :ba-len (alength ^bytes ba) - :data-len (alength ^bytes data-ba)}))) + {:?known-wrapper ?known-wrapper + :?header ?nippy-header + :thawable? (try (thaw unwrapped-ba thaw-opts) true + (catch Exception _ false)) + :unwrapped-ba unwrapped-ba + :data-ba data-ba + :unwrapped-len (alength ^bytes unwrapped-ba) + :ba-len (alength ^bytes ba) + :data-len (alength ^bytes data-ba)})))) -(comment (inspect-ba (freeze "hello")) - (seq (:data-ba (inspect-ba (freeze "hello"))))) +(comment + (inspect-ba (freeze "hello")) + (seq (:data-ba (inspect-ba (freeze "hello"))))) ;;;; Deprecated From c7c0c6fe54426c494d62654d0cabdfb7cd6d9a31 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 30 Sep 2015 12:43:11 +0700 Subject: [PATCH 19/33] Stop documenting `:skip-header?` option It's almost entirely useless now, and dangerous: folks who absolutely know what they're doing can keep using it, but don't broadcast its existance. --- src/taoensso/nippy.clj | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 1991329..7f2ab32 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -400,10 +400,12 @@ "Serializes arg (any Clojure data type) to a byte array. To freeze custom types, extend the Clojure reader or see `extend-freeze`." (^bytes [x] (freeze x nil)) - (^bytes [x {:keys [compressor encryptor password skip-header?] + (^bytes [x {:keys [compressor encryptor password] :or {compressor :auto - encryptor aes128-encryptor}}] - (let [encryptor (when password encryptor) + encryptor aes128-encryptor} + :as opts}] + (let [skip-header? (:skip-header? opts) ; Want this *un*documented + encryptor (when password encryptor) zero-copy-mode? (and (nil? compressor) (nil? encryptor)) baos (ByteArrayOutputStream. 64) dos (DataOutputStream. baos)] From 2df9cb80d6e6a5c56a0bf18203c7260b23adf5cd Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 30 Sep 2015 13:53:48 +0700 Subject: [PATCH 20/33] Add small-bytes type --- src/taoensso/nippy.clj | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 7f2ab32..b18a146 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -71,6 +71,7 @@ (def ^:const id-boolean (byte 4)) (def ^:const id-reader (byte 5)) ; Fallback #2 (def ^:const id-serializable (byte 6)) ; Fallback #1 + (def ^:const id-sm-bytes (byte 7)) (def ^:const id-char (byte 10)) ;; 11 ; Deprecated @@ -176,12 +177,14 @@ (defn byte-sized? [^long n] (<= n 127 #_Byte/MAX_VALUE)) (defn short-sized? [^long n] (<= n 32767 #_Short/MAX_VALUE)) -(defn write-ided-bytes [^DataOutput out id-sm id ^bytes ba] - (if (byte-sized? (alength ba)) - (do (write-id out id-sm) - (write-sm-bytes out ba)) - (do (write-id out id) - (write-bytes out ba)))) +(defn write-ided-bytes + ([ out ba] (write-ided-bytes out id-sm-bytes id-bytes ba)) + ([^DataOutput out id-sm id ^bytes ba] + (if (byte-sized? (alength ba)) + (do (write-id out id-sm) + (write-sm-bytes out ba)) + (do (write-id out id) + (write-bytes out ba))))) (defn write-ided-string [out ^String s] (write-ided-bytes out id-sm-string id-string (.getBytes s "UTF-8"))) @@ -277,9 +280,9 @@ ~@body))) (freezer nil id-nil) -(freezer (Class/forName "[B") id-bytes (write-bytes out x)) (freezer Boolean id-boolean (.writeBoolean out x)) (freezer Character id-char (.writeChar out (int x))) +(freezer* (Class/forName "[B") (write-ided-bytes out x)) (freezer* String (write-ided-string out x)) (freezer* Keyword (write-ided-keyword out x)) @@ -551,15 +554,15 @@ :nippy/unthawable {:class-name class-name :content content}}))) id-nil nil - id-bytes (read-bytes in) id-boolean (.readBoolean in) id-char (.readChar in) - id-string (read-utf8 in) - id-keyword (keyword (read-utf8 in)) + id-bytes (read-bytes in) + id-sm-bytes (read-sm-bytes in) - ;;; Optimized, common-case types (v2.6+) + id-string (read-utf8 in) id-sm-string (read-sm-utf8 in) + id-keyword (keyword (read-utf8 in)) id-sm-keyword (keyword (read-sm-utf8 in)) id-queue (read-coll in (PersistentQueue/EMPTY)) From cf38d6f111b2e487e18b1699fe0007c548f0820a Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 3 Oct 2015 11:26:19 +0700 Subject: [PATCH 21/33] Fix final-freeze-fallback arg order --- src/taoensso/nippy.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index b18a146..c94cee6 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -356,7 +356,7 @@ :else ; Fallback #3: *final-freeze-fallback* (if-let [ffb *final-freeze-fallback*] - (ffb x out) + (ffb out x) (throw (ex-info (format "Unfreezable type: %s %s" (type x) (str x)) {:type (type x) :as-str (enc/pr-edn x)})))))) From e71df20a423a3463c782223e46e7487f3bc2cbb2 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 6 Oct 2015 13:03:33 +0700 Subject: [PATCH 22/33] Tests: re-enable decryption tests with invalid passwords --- test/taoensso/nippy/tests/main.clj | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 964c99e..e7cd462 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -41,11 +41,14 @@ (check-props/for-all [val check-gen/any] (= val (thaw (freeze val))))))) -;;; Trying to decrypt random (invalid) data can actually crash JVM -;; (expect Exception (thaw (freeze test-data {:password "malformed"}))) -;; (expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) -;; (expect Exception (thaw (freeze test-data {:password [:salted "p"]}) -;; {:compressor nil})) +(expect Exception (thaw (freeze test-data {:password "malformed"}))) +(expect Exception (thaw (freeze test-data {:password [:salted "p"]}) + {;; Necessary to prevent against JVM segfault due to + ;; https://goo.gl/t0OUIo: + :v1-compatibility? false})) +(expect Exception (thaw (freeze test-data {:password [:salted "p"]}) + {:v1-compatibility? false ; Ref. https://goo.gl/t0OUIo + :compressor nil})) (expect ; Snappy lib compatibility (for legacy versions of Nippy) (let [^bytes raw-ba (freeze test-data {:compressor nil}) From 9c8adfe5136aec008d22891a9cb23a1c904c1c53 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 6 Oct 2015 13:12:25 +0700 Subject: [PATCH 23/33] **NB BREAKING**: change default `:v1-compatibility?` thaw option Motivation for changing this default: v1 compatibility requires that in the event of a thaw failure, a fallback attempt is made using v1 options. This must include an attempt at Snappy decompression. But the version of Snappy we're using has a major bug that can segfault + crash the JVM when attempted against non-Snappy data: https://github.com/dain/snappy/issues/20 I'd switch to an alternative Snappy implementation, but the only other implementation I'm aware of uses JNI which can introduce troublesome compatibility issues even for people who don't want the Snappy support. Had hoped that the Snappy bug would eventually get fixed, but that's looking unlikely. Nippy v2 was released on July 22nd 2013 (2 years, 2 months ago) - so am hoping that the majority of lib users will no longer have a need for v1 data thaw support at this point. For those that do, they can re-enable v1 thaw support with this flag. If a better alternative solution ever presents (e.g. the Snappy bug is fixed, an alternative implementation turns up, or we write a util to reliably identify Snappy compressed data) - we can re-enable this flag by default. --- src/taoensso/nippy.clj | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index c94cee6..7eb9d46 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -656,19 +656,22 @@ (defn thaw "Deserializes a frozen object from given byte array to its original Clojure - data type. Supports data frozen with current and all previous versions of - Nippy. To thaw custom types, extend the Clojure reader or see `extend-thaw`. + data type. To thaw custom types, extend the Clojure reader or see `extend-thaw`. + + ** By default, supports data frozen with Nippy v2+ ONLY ** + Add `{:v1-compatibility? true}` option to support thawing of data frozen with + legacy versions of Nippy. Options include: + :v1-compatibility? - support data frozen by legacy versions of Nippy? :compressor - An ICompressor, :auto (requires Nippy header), or nil :encryptor - An IEncryptor, :auto (requires Nippy header), or nil" ([ba] (thaw ba nil)) ([^bytes ba {:keys [v1-compatibility? compressor encryptor password] - :or {v1-compatibility? true ; Recommend disabling when possible - compressor :auto - encryptor :auto} + :or {compressor :auto + encryptor :auto} :as opts}] (assert (not (:headerless-meta opts)) From 0905b96ca6e6149ce4e2bce8a5e46848e3a845c5 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 6 Oct 2015 13:32:43 +0700 Subject: [PATCH 24/33] NB: Refactor `thaw` v1 compatibility support --- src/taoensso/nippy.clj | 119 ++++++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 43 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 7eb9d46..c90b3e4 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -407,7 +407,8 @@ :or {compressor :auto encryptor aes128-encryptor} :as opts}] - (let [skip-header? (:skip-header? opts) ; Want this *un*documented + (let [;; Intentionally undocumented: + no-header? (or (:no-header? opts) (:skip-header? opts)) encryptor (when password encryptor) zero-copy-mode? (and (nil? compressor) (nil? encryptor)) baos (ByteArrayOutputStream. 64) @@ -415,7 +416,7 @@ (if zero-copy-mode? (do ; Optimized case - (when-not skip-header? ; Avoid `wrap-header`'s array copy: + (when-not no-header? ; Avoid `wrap-header`'s array copy: (let [head-ba (get-head-ba {:compressor-id nil :encryptor-id nil})] (.write dos head-ba 0 4))) (freeze-to-out! dos x) @@ -427,7 +428,7 @@ compressor (if (identical? compressor :auto) - (if skip-header? + (if no-header? lz4-compressor (*default-freeze-compressor-selector* ba)) (if (fn? compressor) @@ -438,7 +439,7 @@ ba (if compressor (compress compressor ba) ba) ba (if encryptor (encrypt encryptor password ba) ba)] - (if skip-header? + (if no-header? ba (wrap-header ba {:compressor-id (when-let [c compressor] @@ -654,6 +655,12 @@ (throw (ex-info (format "Unrecognized :auto encryptor id: %s" encryptor-id) {:encryptor-id encryptor-id})))) +(def ^:private err-msg-unknown-thaw-failure + "Decryption/decompression failure, or data unfrozen/damaged.") + +(def ^:private err-msg-unrecognized-header + "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?") + (defn thaw "Deserializes a frozen object from given byte array to its original Clojure data type. To thaw custom types, extend the Clojure reader or see `extend-thaw`. @@ -669,7 +676,7 @@ ([ba] (thaw ba nil)) ([^bytes ba - {:keys [v1-compatibility? compressor encryptor password] + {:keys [v1-compatibility? compressor encryptor password no-header?] :or {compressor :auto encryptor :auto} :as opts}] @@ -677,13 +684,17 @@ (assert (not (:headerless-meta opts)) ":headerless-meta `thaw` opt removed in Nippy v2.7+") - (let [ex (fn [msg & [e]] (throw (ex-info (format "Thaw failed: %s" msg) - {:opts (merge opts - {:compressor compressor - :encryptor encryptor})} - e))) + (let [v2+? (not v1-compatibility?) + ex (fn ex + ([ msg] (ex nil msg)) + ([e msg] (throw (ex-info (format "Thaw failed: %s" msg) + {:opts (merge opts + {:compressor compressor + :encryptor encryptor})} + e)))) + thaw-data - (fn [data-ba compressor-id encryptor-id] + (fn [data-ba compressor-id encryptor-id ex-fn] (let [compressor (if (identical? compressor :auto) (get-auto-compressor compressor-id) compressor) @@ -701,43 +712,65 @@ dis (DataInputStream. (ByteArrayInputStream. ba))] (thaw-from-in! dis)) - (catch Exception e - (ex "Decryption/decompression failure, or data unfrozen/damaged." - e))))) + (catch Exception e (ex-fn e))))) ;; This is hackish and can actually currently result in JVM core dumps - ;; due to buggy Snappy behaviour, Ref. http://goo.gl/mh7Rpy. - thaw-nippy-v1-data - (fn [data-ba] - (if-not v1-compatibility? - (throw (Exception. "v1 compatibility disabled")) - (try (thaw-data data-ba :snappy nil) - (catch Exception _ - (thaw-data data-ba nil nil)))))] + ;; due to buggy Snappy behaviour, Ref. http://goo.gl/mh7Rpy + thaw-legacy-data + (fn [data-ba ex-fn] + (thaw-data data-ba :snappy nil + (fn [_] (thaw-data data-ba nil nil + (fn [_] (ex-fn nil))))))] - (if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?] - :as head-meta}] (try-parse-header ba)] + (if no-header? + (if v2+? + (thaw-data ba :no-header :no-header + (fn [e] (ex e err-msg-unknown-thaw-failure))) - ;; A well-formed header _appears_ to be present (it's possible though - ;; unlikely that this is a fluke and data is actually headerless): - (try (thaw-data data-ba compressor-id encryptor-id) - (catch Exception e - (try (thaw-nippy-v1-data data-ba) - (catch Exception _ - (if unrecognized-meta? - (ex "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?" - e) - (throw e)))))) + (thaw-data ba :no-header :no-header + (fn [e] + (thaw-legacy-data ba + (fn [_] (ex e err-msg-unknown-thaw-failure)))))) - ;; Well-formed header definitely not present - (try (thaw-nippy-v1-data ba) - (catch Exception _ - (thaw-data ba :no-header :no-header))))))) + (if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?] + :as head-meta}] (try-parse-header ba)] -(comment (thaw (freeze "hello")) - (thaw (freeze "hello" {:compressor nil})) - (thaw (freeze "hello" {:password [:salted "p"]})) ; ex: no pwd - (thaw (freeze "hello") {:password [:salted "p"]})) + ;; A well-formed header _appears_ to be present (it's possible though + ;; unlikely that this is a fluke and data is actually headerless): + (if v2+? + (thaw-data data-ba compressor-id encryptor-id + (fn [e] + (thaw-data ba :no-header :no-header + (fn [_] + (if unrecognized-meta? + (ex e err-msg-unrecognized-header) + (ex e err-msg-unknown-thaw-failure)))))) + + (thaw-data data-ba compressor-id encryptor-id + (fn [e] + (thaw-data ba :no-header :no-header + (fn [_] + (thaw-legacy-data ba + (fn [_] + (if unrecognized-meta? + (ex e err-msg-unrecognized-header) + (ex e err-msg-unknown-thaw-failure))))))))) + + ;; Well-formed header definitely not present + (if v2+? + (thaw-data ba :no-header :no-header + (fn [e] (ex e err-msg-unknown-thaw-failure))) + + (thaw-data ba :no-header :no-header + (fn [e] + (thaw-legacy-data ba + (fn [_] (ex e err-msg-unknown-thaw-failure))))))))))) + +(comment + (thaw (freeze "hello")) + (thaw (freeze "hello" {:compressor nil})) + (thaw (freeze "hello" {:password [:salted "p"]})) ; ex: no pwd + (thaw (freeze "hello") {:password [:salted "p"]})) ;;;; Custom types @@ -922,7 +955,7 @@ ;; Deprecated by :auto compressor selection (defrecord Compressable-LZMA2 [value]) ; Why was this `LZMA2` instead of `lzma2`? (extend-freeze Compressable-LZMA2 128 [x out] - (let [ba (freeze (:value x) {:skip-header? true :compressor nil}) + (let [ba (freeze (:value x) {:no-header? true :compressor nil}) ba-len (alength ba) compress? (> ba-len 1024)] (.writeBoolean out compress?) From 037cb147392238336bac86333bfa62b06ec32f3d Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 6 Oct 2015 15:04:19 +0700 Subject: [PATCH 25/33] Misc hk --- src/taoensso/nippy.clj | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index c90b3e4..1b4d768 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -408,13 +408,12 @@ encryptor aes128-encryptor} :as opts}] (let [;; Intentionally undocumented: - no-header? (or (:no-header? opts) (:skip-header? opts)) - encryptor (when password encryptor) - zero-copy-mode? (and (nil? compressor) (nil? encryptor)) + no-header? (or (:no-header? opts) (:skip-header? opts)) + encryptor (when password encryptor) baos (ByteArrayOutputStream. 64) dos (DataOutputStream. baos)] - (if zero-copy-mode? + (if (and (nil? compressor) (nil? encryptor)) (do ; Optimized case (when-not no-header? ; Avoid `wrap-header`'s array copy: (let [head-ba (get-head-ba {:compressor-id nil :encryptor-id nil})] @@ -442,12 +441,17 @@ (if no-header? ba (wrap-header ba - {:compressor-id (when-let [c compressor] - (or (compression/standard-header-ids - (compression/header-id c)) :else)) - :encryptor-id (when-let [e encryptor] - (or (encryption/standard-header-ids - (encryption/header-id e)) :else))})))))))) + {:compressor-id + (when-let [c compressor] + (or (compression/standard-header-ids + (compression/header-id c)) + :else)) + + :encryptor-id + (when-let [e encryptor] + (or (encryption/standard-header-ids + (encryption/header-id e)) + :else))})))))))) ;;;; Thawing @@ -671,8 +675,8 @@ Options include: :v1-compatibility? - support data frozen by legacy versions of Nippy? - :compressor - An ICompressor, :auto (requires Nippy header), or nil - :encryptor - An IEncryptor, :auto (requires Nippy header), or nil" + :compressor - :auto (checks header, default) an ICompressor, or nil + :encryptor - :auto (checks header, default), an IEncryptor, or nil" ([ba] (thaw ba nil)) ([^bytes ba @@ -785,7 +789,7 @@ (assert-custom-type-id custom-type-id) (if-not (keyword? custom-type-id) (int (- ^long custom-type-id)) - (let [^long hash-id (hash custom-type-id) + (let [^int hash-id (hash custom-type-id) short-hash-id (if (pos? hash-id) (mod hash-id Short/MAX_VALUE) (mod hash-id Short/MIN_VALUE))] From 3479ddad00859557ea97468f6e0065c17b5d8238 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 6 Oct 2015 15:07:37 +0700 Subject: [PATCH 26/33] Clean up `thaw` fallback behaviour, decrease number of fallback cases --- src/taoensso/nippy.clj | 73 ++++++++++++------------------ test/taoensso/nippy/tests/main.clj | 5 +- 2 files changed, 32 insertions(+), 46 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 1b4d768..27f1ac3 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -680,7 +680,7 @@ ([ba] (thaw ba nil)) ([^bytes ba - {:keys [v1-compatibility? compressor encryptor password no-header?] + {:keys [v1-compatibility? compressor encryptor password] :or {compressor :auto encryptor :auto} :as opts}] @@ -688,14 +688,15 @@ (assert (not (:headerless-meta opts)) ":headerless-meta `thaw` opt removed in Nippy v2.7+") - (let [v2+? (not v1-compatibility?) - ex (fn ex - ([ msg] (ex nil msg)) - ([e msg] (throw (ex-info (format "Thaw failed: %s" msg) - {:opts (merge opts - {:compressor compressor - :encryptor encryptor})} - e)))) + (let [v2+? (not v1-compatibility?) + no-header? (:no-header? opts) ; Intentionally undocumented + ex (fn ex + ([ msg] (ex nil msg)) + ([e msg] (throw (ex-info (format "Thaw failed: %s" msg) + {:opts (merge opts + {:compressor compressor + :encryptor encryptor})} + e)))) thaw-data (fn [data-ba compressor-id encryptor-id ex-fn] @@ -718,57 +719,40 @@ (catch Exception e (ex-fn e))))) - ;; This is hackish and can actually currently result in JVM core dumps - ;; due to buggy Snappy behaviour, Ref. http://goo.gl/mh7Rpy - thaw-legacy-data + ;; Hackish + can actually segfault JVM due to Snappy bug, + ;; Ref. http://goo.gl/mh7Rpy - no better alternatives, unfortunately + thaw-v1-data (fn [data-ba ex-fn] (thaw-data data-ba :snappy nil - (fn [_] (thaw-data data-ba nil nil - (fn [_] (ex-fn nil))))))] + (fn [_] (thaw-data data-ba nil nil (fn [_] (ex-fn nil))))))] (if no-header? (if v2+? + (thaw-data ba :no-header :no-header (fn [e] (ex e err-msg-unknown-thaw-failure))) (thaw-data ba :no-header :no-header - (fn [e] (ex e err-msg-unknown-thaw-failure))) - - (thaw-data ba :no-header :no-header - (fn [e] - (thaw-legacy-data ba - (fn [_] (ex e err-msg-unknown-thaw-failure)))))) + (fn [e] (thaw-v1-data ba (fn [_] (ex e err-msg-unknown-thaw-failure)))))) + ;; At this point we assume that we have a header iff we have v2+ data (if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?] :as head-meta}] (try-parse-header ba)] ;; A well-formed header _appears_ to be present (it's possible though ;; unlikely that this is a fluke and data is actually headerless): (if v2+? - (thaw-data data-ba compressor-id encryptor-id - (fn [e] - (thaw-data ba :no-header :no-header - (fn [_] - (if unrecognized-meta? - (ex e err-msg-unrecognized-header) - (ex e err-msg-unknown-thaw-failure)))))) + (if unrecognized-meta? + (ex err-msg-unrecognized-header) + (thaw-data data-ba compressor-id encryptor-id + (fn [e] (ex e err-msg-unknown-thaw-failure)))) - (thaw-data data-ba compressor-id encryptor-id - (fn [e] - (thaw-data ba :no-header :no-header - (fn [_] - (thaw-legacy-data ba - (fn [_] - (if unrecognized-meta? - (ex e err-msg-unrecognized-header) - (ex e err-msg-unknown-thaw-failure))))))))) + (if unrecognized-meta? + (thaw-v1-data ba (fn [_] (ex err-msg-unrecognized-header))) + (thaw-data data-ba compressor-id encryptor-id + (fn [e] (thaw-v1-data ba (fn [_] (ex e err-msg-unknown-thaw-failure))))))) ;; Well-formed header definitely not present (if v2+? - (thaw-data ba :no-header :no-header - (fn [e] (ex e err-msg-unknown-thaw-failure))) - - (thaw-data ba :no-header :no-header - (fn [e] - (thaw-legacy-data ba - (fn [_] (ex e err-msg-unknown-thaw-failure))))))))))) + (ex err-msg-unknown-thaw-failure) + (thaw-v1-data ba (fn [_] (ex err-msg-unknown-thaw-failure))))))))) (comment (thaw (freeze "hello")) @@ -970,7 +954,8 @@ (extend-thaw 128 [in] (let [compressed? (.readBoolean in) ba (read-bytes in)] - (thaw ba {:compressor (when compressed? lzma2-compressor) + (thaw ba {:no-header? true + :compressor (when compressed? lzma2-compressor) :encryptor nil}))) (comment diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index e7cd462..a43c2cf 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -17,9 +17,10 @@ (expect (do (println (str "Clojure version: " *clojure-version*)) true)) (expect test-data ((comp thaw freeze) test-data)) -(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor +(expect test-data ((comp #(thaw % {:no-header? true + :compressor nippy/lz4-compressor :encryptor nil}) - #(freeze % {:skip-header? true})) + #(freeze % {:no-header? true})) test-data)) (expect test-data ((comp #(thaw % {:password [:salted "p"]}) #(freeze % {:password [:salted "p"]})) From 4df5446c5bb1f13230477e1bc0a481ce48d00e70 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 6 Oct 2015 14:37:08 +0700 Subject: [PATCH 27/33] Update benchmarks --- src/taoensso/nippy/benchmarks.clj | 46 ++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 30860c9..50e81f9 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -4,7 +4,7 @@ [taoensso.encore :as enc] [taoensso.nippy :as nippy :refer (freeze thaw)])) -(def data nippy/stress-data-benchable) +(def data #_22 nippy/stress-data-benchable) (defn fressian-freeze [value] (let [^java.nio.ByteBuffer bb (fressian/write value) @@ -39,21 +39,24 @@ (println {:reader (bench1 enc/pr-edn enc/read-edn #(count (.getBytes ^String % "UTF-8")))})) - (println {:default (bench1 #(freeze % {}) - #(thaw % {}))}) - (println {:fast (bench1 #(freeze % {:compressor nil - :skip-header? true}) - #(thaw % {:compressor nil - :encryptor nil}))}) - (println {:encrypted (bench1 #(freeze % {:password [:cached "p"]}) - #(thaw % {:password [:cached "p"]}))}) - - (when lzma2? ; Slow as molasses + (when lzma2? ; Slow (println {:lzma2 (bench1 #(freeze % {:compressor nippy/lzma2-compressor}) - #(thaw % {:compressor nippy/lzma2-compressor}))})) + #(thaw % {:compressor nippy/lzma2-compressor}))})) (when fressian? - (println {:fressian (bench1 fressian-freeze fressian-thaw)}))) + (println {:fressian (bench1 fressian-freeze fressian-thaw)})) + + (println {:encrypted (bench1 #(freeze % {:password [:cached "p"]}) + #(thaw % {:password [:cached "p"]}))}) + (println {:default (bench1 #(freeze % {}) + #(thaw % {}))}) + (println {:fast1 (bench1 #(freeze % {:compressor nil}) + #(thaw % {:compressor nil}))}) + (println {:fast2 (bench1 #(freeze % {:no-header? true + :compressor nil}) + #(thaw % {:no-header? true + :compressor nil + :encryptor nil}))})) (println "\nDone! (Time for cake?)") true) @@ -68,6 +71,23 @@ ;; (bench {:laps 2 :lzma2? true}) ;; (bench {:laps 2}) + ;;; 2015 Oct 6, v2.11.0-alpha4 + {:reader {:round 73409, :freeze 21823, :thaw 51586, :size 27672}} + {:lzma2 {:round 56689, :freeze 37222, :thaw 19467, :size 11252}} + {:fressian {:round 10666, :freeze 7737, :thaw 2929, :size 16985}} + {:encrypted {:round 6885, :freeze 4227, :thaw 2658, :size 16148}} + {:default {:round 6304, :freeze 3824, :thaw 2480, :size 16122}} + {:fast1 {:round 5352, :freeze 3272, :thaw 2080, :size 16976}} + {:fast2 {:round 5243, :freeze 3238, :thaw 2005, :size 16972}} + ;; + {:reader {:round 26, :freeze 17, :thaw 9, :size 2}} + {:lzma2 {:round 3648, :freeze 3150, :thaw 498, :size 68}} + {:fressian {:round 19, :freeze 7, :thaw 12, :size 1}} + {:encrypted {:round 63, :freeze 40, :thaw 23, :size 36}} + {:default {:round 24, :freeze 17, :thaw 7, :size 6}} + {:fast1 {:round 19, :freeze 12, :thaw 7, :size 6}} + {:fast2 {:round 4, :freeze 2, :thaw 2, :size 2}} + ;;; 2015 Sep 29, after read/write API refactor {:lzma2 {:round 51640, :freeze 33699, :thaw 17941, :size 11240}} {:encrypted {:round 5922, :freeze 3734, :thaw 2188, :size 16132}} From f59f2f33cbe78a914bcace2698c8311d1c159c57 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 1 Dec 2015 13:38:44 +0700 Subject: [PATCH 28/33] NB fix min-val int-as-long --- src/taoensso/nippy.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 27f1ac3..813e2ca 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -205,8 +205,8 @@ (do (write-id out id-short-as-long) (.writeShort out n)) - (and (<= n 2147483647 #_Integer/MAX_VALUE) - (>= -2147483648 #_Integer/MIN_VALUE)) + (and (<= n 2147483647 #_Integer/MAX_VALUE) + (>= n -2147483648 #_Integer/MIN_VALUE)) (do (write-id out id-int-as-long) (.writeInt out n)) From f70cfc37720788407e4a25c6fb1170ff82a635e3 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 1 Dec 2015 13:00:53 +0700 Subject: [PATCH 29/33] Bump deps --- project.clj | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/project.clj b/project.clj index 2fc5db4..ee71d7b 100644 --- a/project.clj +++ b/project.clj @@ -9,12 +9,13 @@ :min-lein-version "2.3.3" :global-vars {*warn-on-reflection* true *assert* true - *unchecked-math* false} + ;; *unchecked-math* :warn-on-boxed + } :dependencies [[org.clojure/clojure "1.5.1"] - [org.clojure/tools.reader "0.9.2"] - [com.taoensso/encore "2.18.0"] + [org.clojure/tools.reader "0.10.0"] + [com.taoensso/encore "2.26.1"] [org.iq80.snappy/snappy "0.4"] [org.tukaani/xz "1.5"] [net.jpountz.lz4/lz4 "1.3"]] @@ -25,22 +26,24 @@ :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} - :1.8 {:dependencies [[org.clojure/clojure "1.8.0-alpha5"]]} + :1.8 {:dependencies [[org.clojure/clojure "1.8.0-RC2"]]} :test {:jvm-opts ["-Xms1024m" "-Xmx2048m"] - :dependencies [[expectations "2.1.1"] - [org.clojure/test.check "0.8.2"] + :dependencies [[expectations "2.1.4"] + [org.clojure/test.check "0.9.0"] [org.clojure/data.fressian "0.2.1"] [org.xerial.snappy/snappy-java "1.1.2"]]} :dev [:1.7 :test {:plugins - [[lein-pprint "1.1.1"] - [lein-ancient "0.6.7"] - [lein-expectations "0.0.8"] - [lein-autoexpect "1.2.2"] - [codox "0.8.10"]]}]} + [[lein-pprint "1.1.2"] + [lein-ancient "0.6.8"] + [lein-codox "0.9.0"]]}]} :test-paths ["test" "src"] + :codox + {:language :clojure ; [:clojure :clojurescript] ; No support? + :source-uri "https://github.com/ptaoussanis/nippy/blob/master/{filepath}#L{line}"} + :aliases {"test-all" ["with-profile" "+1.5:+1.6:+1.7:+1.8" "expectations"] "test-auto" ["with-profile" "+test" "autoexpect"] From 5849320d3a5cccebd540365b49e34cc944632199 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 1 Dec 2015 13:07:38 +0700 Subject: [PATCH 30/33] Drop Expectations, migrate to clojure.test, update test.check stuff --- project.clj | 6 +- test/taoensso/nippy/tests/main.clj | 285 ++++++++++++++++------------- 2 files changed, 158 insertions(+), 133 deletions(-) diff --git a/project.clj b/project.clj index ee71d7b..dc2bf72 100644 --- a/project.clj +++ b/project.clj @@ -28,8 +28,7 @@ :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} :1.8 {:dependencies [[org.clojure/clojure "1.8.0-RC2"]]} :test {:jvm-opts ["-Xms1024m" "-Xmx2048m"] - :dependencies [[expectations "2.1.4"] - [org.clojure/test.check "0.9.0"] + :dependencies [[org.clojure/test.check "0.9.0"] [org.clojure/data.fressian "0.2.1"] [org.xerial.snappy/snappy-java "1.1.2"]]} :dev [:1.7 :test @@ -45,8 +44,7 @@ :source-uri "https://github.com/ptaoussanis/nippy/blob/master/{filepath}#L{line}"} :aliases - {"test-all" ["with-profile" "+1.5:+1.6:+1.7:+1.8" "expectations"] - "test-auto" ["with-profile" "+test" "autoexpect"] + {"test-all" ["with-profile" "+1.5:+1.6:+1.7:+1.8" "test"] "deploy-lib" ["do" "deploy" "clojars," "install"] "start-dev" ["with-profile" "+server-jvm" "repl" ":headless"]} diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index a43c2cf..906d174 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -1,174 +1,201 @@ (ns taoensso.nippy.tests.main - (:require [clojure.test.check :as check] - [clojure.test.check.generators :as check-gen] - [clojure.test.check.properties :as check-props] - [expectations :as test :refer :all] - [taoensso.nippy :as nippy :refer (freeze thaw)] - [taoensso.nippy.benchmarks :as benchmarks])) + (:require + [clojure.test :as test :refer (is are deftest run-tests)] + [clojure.test.check :as tc] + [clojure.test.check.generators :as tc-gens] + [clojure.test.check.properties :as tc-props] + [taoensso.encore :as enc :refer ()] + [taoensso.nippy :as nippy :refer (freeze thaw)] + [taoensso.nippy.benchmarks :as benchmarks])) -(comment (test/run-tests '[taoensso.nippy.tests.main])) +(comment (test/run-tests)) (def test-data nippy/stress-data-comparable) -(defn- before-run {:expectations-options :before-run} []) -(defn- after-run {:expectations-options :after-run} []) +(def tc-num-tests 120) +(def tc-gens + "Like `tc-gens/any` but removes NaN (which breaks equality tests)" + (tc-gens/recursive-gen tc-gens/container-type #_simple-type + (tc-gens/one-of + [tc-gens/int tc-gens/large-integer #_tc-gens/double + (tc-gens/double* {:NaN? false}) + tc-gens/char tc-gens/string tc-gens/ratio tc-gens/boolean tc-gens/keyword + tc-gens/keyword-ns tc-gens/symbol tc-gens/symbol-ns tc-gens/uuid]))) + +(comment (tc-gens/sample tc-gens 10)) ;;;; Core -(expect (do (println (str "Clojure version: " *clojure-version*)) true)) +(deftest _core + (is (do (println (str "Clojure version: " *clojure-version*)) true)) + (is (= test-data ((comp thaw freeze) test-data))) + (is (= test-data ((comp #(thaw % {:no-header? true + :compressor nippy/lz4-compressor + :encryptor nil}) + #(freeze % {:no-header? true})) + test-data))) -(expect test-data ((comp thaw freeze) test-data)) -(expect test-data ((comp #(thaw % {:no-header? true - :compressor nippy/lz4-compressor - :encryptor nil}) - #(freeze % {:no-header? true})) - test-data)) -(expect test-data ((comp #(thaw % {:password [:salted "p"]}) - #(freeze % {:password [:salted "p"]})) - test-data)) -(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor}) - #(freeze % {:compressor nippy/lzma2-compressor})) - test-data)) -(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor - :password [:salted "p"]}) - #(freeze % {:compressor nippy/lzma2-compressor - :password [:salted "p"]})) - test-data)) -(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor}) - #(freeze % {:compressor nippy/lz4hc-compressor})) - test-data)) + (is (= test-data ((comp #(thaw % {:password [:salted "p"]}) + #(freeze % {:password [:salted "p"]})) + test-data))) -(expect ; Try roundtrip anything that simple-check can dream up - (:result (check/quick-check 80 ; Time is n-non-linear - (check-props/for-all [val check-gen/any] - (= val (thaw (freeze val))))))) + (is (= test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor}) + #(freeze % {:compressor nippy/lzma2-compressor})) + test-data))) -(expect Exception (thaw (freeze test-data {:password "malformed"}))) -(expect Exception (thaw (freeze test-data {:password [:salted "p"]}) - {;; Necessary to prevent against JVM segfault due to - ;; https://goo.gl/t0OUIo: - :v1-compatibility? false})) -(expect Exception (thaw (freeze test-data {:password [:salted "p"]}) - {:v1-compatibility? false ; Ref. https://goo.gl/t0OUIo - :compressor nil})) + (is (= test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor + :password [:salted "p"]}) + #(freeze % {:compressor nippy/lzma2-compressor + :password [:salted "p"]})) + test-data))) -(expect ; Snappy lib compatibility (for legacy versions of Nippy) - (let [^bytes raw-ba (freeze test-data {:compressor nil}) - ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) - ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] - (= (thaw raw-ba) - (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) - (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) - (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) - (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) + (is (= test-data ((comp #(thaw % {:compressor nippy/lz4-compressor}) + #(freeze % {:compressor nippy/lz4hc-compressor})) + test-data))) + + (is ; Try roundtrip anything that simple-check can dream up + (:result (tc/quick-check tc-num-tests + (tc-props/for-all [val tc-gens] + (= val (thaw (freeze val))))))) + + (is (thrown? Exception (thaw (freeze test-data {:password "malformed"})))) + (is (thrown? Exception (thaw (freeze test-data {:password [:salted "p"]}) + {;; Necessary to prevent against JVM segfault due to + ;; https://goo.gl/t0OUIo: + :v1-compatibility? false}))) + (is (thrown? Exception (thaw (freeze test-data {:password [:salted "p"]}) + {:v1-compatibility? false ; Ref. https://goo.gl/t0OUIo + :compressor nil}))) + + (is ; Snappy lib compatibility (for legacy versions of Nippy) + (let [^bytes raw-ba (freeze test-data {:compressor nil}) + ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) + ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] + (= (thaw raw-ba) + (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) + (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) + (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) + (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))) ;;;; Custom types & records -;;; Extend to custom Type -(defrecord MyType [data]) -(expect Exception (do (nippy/extend-freeze MyType 1 [x s] (.writeUTF s (:data x))) - (thaw (freeze (->MyType "val"))))) -(expect (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s))) - (let [type (->MyType "val")] (= type (thaw (freeze type)))))) +(deftype MyType [data]) +(defrecord MyRec [data]) -;;; Extend to custom Record -(defrecord MyRec [data]) -(expect (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "foo-" (:data x)))) - (nippy/extend-thaw 2 [s] (->MyRec (.readUTF s))) - (= (->MyRec "foo-val") (thaw (freeze (->MyRec "val")))))) +(deftest _types + ;;; Extend to custom Type + (is (thrown? Exception ; No thaw extension yet + (do (nippy/swap-custom-readers! (constantly {})) + (nippy/extend-freeze MyType 1 [x s] (.writeUTF s (.data x))) + (thaw (freeze (->MyType "val")))))) + (is (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s))) + (let [mt (->MyType "val")] (= (.data ^MyType mt) + (.data ^MyType (thaw (freeze mt))))))) -;;; Keyword (prefixed) extensions -(expect - (do (nippy/extend-freeze MyType :nippy-tests/MyType [x s] (.writeUTF s (:data x))) - (nippy/extend-thaw :nippy-tests/MyType [s] (->MyType (.readUTF s))) - (let [type (->MyType "val")] (= type (thaw (freeze type)))))) + ;;; Extend to custom Record + (is (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "foo-" (:data x)))) + (nippy/extend-thaw 2 [s] (->MyRec (.readUTF s))) + (= (->MyRec "foo-val") (thaw (freeze (->MyRec "val")))))) + + ;;; Keyword (prefixed) extensions + (is + (do (nippy/extend-freeze MyRec :nippy-tests/MyRec [x s] (.writeUTF s (:data x))) + (nippy/extend-thaw :nippy-tests/MyRec [s] (->MyRec (.readUTF s))) + (let [mr (->MyRec "val")] (= mr (thaw (freeze mr))))))) ;;;; Stable binary representation of vals -(expect (seq (freeze test-data)) - (seq (freeze test-data))) ; f(x)=f(y) | x=y +(deftest _stable-bin -;; As above, but try multiple times to catch possible protocol interface races: -(expect #(every? true? %) + (is (= (seq (freeze test-data)) + (seq (freeze test-data)))) ; f(x)=f(y) | x=y + + ;; As above, but try multiple times to catch possible protocol interface races: + (is (every? true? (repeatedly 1000 (fn [] (= (seq (freeze test-data)) - (seq (freeze test-data)))))) + (seq (freeze test-data))))))) -;; NB abandoning - no way to do this reliably w/o appropriate contracts from -;; (seq ): -;; -;; (expect (seq (-> test-data freeze)) ; f(x)=f(f-1(f(x))) -;; (seq (-> test-data freeze thaw freeze))) -;; -;; As above, but with repeated refreeze to catch possible protocol interface races: -;; (expect (= (seq (freeze test-data)) -;; (seq (reduce (fn [frozen _] (freeze (thaw frozen))) -;; (freeze test-data) (range 1000))))) + ;; NB abandoning - no way to do this reliably w/o appropriate contracts from + ;; (seq ): + ;; + ;; (is (= (seq (-> test-data freeze)) + ;; (seq (-> test-data freeze thaw freeze)))) ; f(x)=f(f-1(f(x))) + ;; + ;; As above, but with repeated refreeze to catch possible protocol interface races: + ;; (is (= (seq (freeze test-data)) + ;; (seq (reduce (fn [frozen _] (freeze (thaw frozen))) + ;; (freeze test-data) (range 1000))))) + ) (defn qc-prop-bijection [& [n]] (let [bin->val (atom {}) val->bin (atom {})] (merge - (check/quick-check (or n 1) - (check-props/for-all [val check-gen/any #_check-gen/any-printable] - (let [;; Nb need `seq` for Clojure hash equality: - bin (hash (seq (freeze val)))] - (and - (if (contains? val->bin val) - (= (get val->bin val) bin) ; x=y => f(x)=f(y) by clj= - (do (swap! val->bin assoc val bin) - true)) + (tc/quick-check (or n 1) + (tc-props/for-all [val tc-gens] + (let [;; Nb need `seq` for Clojure hash equality: + bin (hash (seq (freeze val)))] + (and + (if (contains? val->bin val) + (= (get val->bin val) bin) ; x=y => f(x)=f(y) by clj= + (do (swap! val->bin assoc val bin) + true)) - (if (contains? bin->val bin) - (= (get bin->val bin) val) ; f(x)=f(y) => x=y by clj= - (do (swap! bin->val assoc bin val) - true)))))) - #_{:bin->val @bin->val - :val->bin @val->bin} - nil))) + (if (contains? bin->val bin) + (= (get bin->val bin) val) ; f(x)=f(y) => x=y by clj= + (do (swap! bin->val assoc bin val) + true)))))) + #_{:bin->val @bin->val + :val->bin @val->bin} + nil))) (comment - (check-gen/sample check-gen/any 10) + (tc-gens/sample tc-gens 10) (:result (qc-prop-bijection 80)) (let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)] [result (vals bin->val)])) -(expect #(:result %) (qc-prop-bijection 80)) +(deftest _gc-prop-bijection + (is (:result (qc-prop-bijection tc-num-tests)))) ;;;; Thread safety ;; Not sure why, but record equality test fails in futures: (def test-data-threaded (dissoc nippy/stress-data-comparable :stress-record)) -(expect - (let [futures - (mapv - (fn [_] - (future - (= (thaw (freeze test-data-threaded)) test-data-threaded))) - (range 50))] - (every? deref futures))) +(deftest _thread-safe + (is + (let [futures + (mapv + (fn [_] + (future + (= (thaw (freeze test-data-threaded)) test-data-threaded))) + (range 50))] + (every? deref futures))) -(expect - (let [futures - (mapv - (fn [_] - (future - (= (thaw (freeze test-data-threaded {:password [:salted "password"]}) - {:password [:salted "password"]}) - test-data-threaded))) - (range 50))] - (every? deref futures))) + (is + (let [futures + (mapv + (fn [_] + (future + (= (thaw (freeze test-data-threaded {:password [:salted "password"]}) + {:password [:salted "password"]}) + test-data-threaded))) + (range 50))] + (every? deref futures))) -(expect - (let [futures - (mapv - (fn [_] - (future - (= (thaw (freeze test-data-threaded {:password [:cached "password"]}) - {:password [:cached "password"]}) - test-data-threaded))) - (range 50))] - (every? deref futures))) + (is + (let [futures + (mapv + (fn [_] + (future + (= (thaw (freeze test-data-threaded {:password [:cached "password"]}) + {:password [:cached "password"]}) + test-data-threaded))) + (range 50))] + (every? deref futures)))) ;;;; Benchmarks -(expect (benchmarks/bench {})) ; Also tests :cached passwords +(deftest _benchmarks + (is (benchmarks/bench {})) ; Also tests :cached passwords + ) From 643d762bbeecdc855bd52a00f493c9d26f610eb5 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 1 Dec 2015 15:20:29 +0700 Subject: [PATCH 31/33] Fix :auto encryption unit tests --- src/taoensso/nippy/encryption.clj | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/taoensso/nippy/encryption.clj b/src/taoensso/nippy/encryption.clj index eb946dc..3f1037e 100644 --- a/src/taoensso/nippy/encryption.clj +++ b/src/taoensso/nippy/encryption.clj @@ -76,13 +76,14 @@ (defrecord AES128Encryptor [key-gen key-cache] IEncryptor - (header-id [_] (if (= key-gen sha512-key) :aes128-sha512 :aes128-other)) + (header-id [_] (if (identical? key-gen :sha512) :aes128-sha512 :aes128-other)) (encrypt [_ typed-pwd data-ba] (let [[type pwd] (destructure-typed-pwd typed-pwd) salt? (identical? type :salted) iv-ba (rand-bytes aes128-block-size) salt-ba (when salt? (rand-bytes salt-size)) prefix-ba (if-not salt? iv-ba (enc/ba-concat iv-ba salt-ba)) + key-gen (if (identical? key-gen :sha512) sha512-key key-gen) key (if salt? (key-gen salt-ba pwd) (enc/memoized key-cache key-gen salt-ba pwd)) @@ -98,7 +99,8 @@ prefix-size (+ aes128-block-size (if salt? salt-size 0)) [prefix-ba data-ba] (enc/ba-split ba prefix-size) [iv-ba salt-ba] (if-not salt? [prefix-ba nil] - (enc/ba-split prefix-ba aes128-block-size)) + (enc/ba-split prefix-ba aes128-block-size)) + key-gen (if (identical? key-gen :sha512) sha512-key key-gen) key (if salt? (key-gen salt-ba pwd) (enc/memoized key-cache key-gen salt-ba pwd)) @@ -143,7 +145,7 @@ Faster than `aes128-salted`, and harder to attack any particular key - but increased danger if a key is somehow compromised." - (->AES128Encryptor sha512-key (atom {}))) + (->AES128Encryptor :sha512 (atom {}))) ;;;; Default implementation From c483e157bd837564ad3ea5ea2dc3b61cee2e72a2 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 1 Dec 2015 15:30:13 +0700 Subject: [PATCH 32/33] Encryption ns housekeeping --- src/taoensso/nippy/encryption.clj | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/taoensso/nippy/encryption.clj b/src/taoensso/nippy/encryption.clj index 3f1037e..d8b72cb 100644 --- a/src/taoensso/nippy/encryption.clj +++ b/src/taoensso/nippy/encryption.clj @@ -1,12 +1,11 @@ (ns taoensso.nippy.encryption - "Simple no-nonsense crypto with reasonable defaults. Because your Clojure data - deserves some privacy." + "Simple no-nonsense crypto with reasonable defaults." {:author "Peter Taoussanis"} (:require [taoensso.encore :as enc])) ;;;; Interface -(def standard-header-ids "These'll support :auto thaw." #{:aes128-sha512}) +(def standard-header-ids "These'll support :auto thaw" #{:aes128-sha512}) (defprotocol IEncryptor (header-id [encryptor]) @@ -40,11 +39,13 @@ (defn- sha512-key "SHA512-based key generator. Good JVM availability without extra dependencies - (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds." + (PBKDF2, bcrypt, scrypt, etc.). Decent security when using many rounds." + ;; [salt-ba ^String pwd & [n]] [salt-ba ^String pwd] (let [md (sha512-md)] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] (if salt-ba (enc/ba-concat salt-ba pwd-ba) pwd-ba)) + ;; n (or n (* (int Short/MAX_VALUE) (if salt-ba 5 64))) n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] (if-not (zero? n) (recur (.digest md ba) (dec n)) @@ -111,7 +112,7 @@ (.doFinal cipher data-ba)))) (def aes128-encryptor - "Default 128bit AES encryptor with multi-round SHA-512 key-gen. + "Default 128bit AES encryptor with many-round SHA-512 key-gen. Password form [:salted \"my-password\"] --------------------------------------- From d129da990cf149b757cbae9109c5c7db90d8f289 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 1 Dec 2015 15:33:00 +0700 Subject: [PATCH 33/33] v2.11.0-alpha1 --- CHANGELOG.md | 18 ++++++++++++++++++ README.md | 3 ++- project.clj | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b7f385..53148ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ > This project uses [Break Versioning](https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md) as of **Aug 16, 2014**. +## v2.11.0-alpha1 / 2015 Dec 1 + +> This is a major performance release that **drops default support for thawing Nippy v1 archives** but is otherwise non-breaking + +* **BREAKING**: `thaw` now has `:v1-compatibility?` opt set to false by default (was true before) [1] +* **Performance**: optimize serialized size of small maps, sets, vectors, bytes +* **Performance**: optimized (no copy) `freeze` when using no compression or encryption +* **Implementation**: swap most macros for fns (make low-level utils easier to use) + +```clojure +[com.taoensso/nippy "2.11.0-alpha1"] +``` + +#### Notes + +**[1]** Use `(thaw {:v1-compatibility? true})` to support thawing of data frozen with Nippy v1 (before ~June 2013) + + ## v2.10.0 / 2015 Sep 30 > This is a major feature/performance release that **drops support for Clojure 1.4** but is otherwise non-breaking diff --git a/README.md b/README.md index 2223c63..a12b3ce 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ **[API docs][]** | **[CHANGELOG][]** | [other Clojure libs][] | [Twitter][] | [contact/contrib](#contact--contributing) | current [Break Version][]: ```clojure -[com.taoensso/nippy "2.10.0"] ; Stable, see CHANGELOG for details +[com.taoensso/nippy "2.10.0"] ; Stable +[com.taoensso/nippy "2.11.0-alpha1"] ; Dev, see CHANGELOG for details ``` # Nippy, a Clojure serialization library diff --git a/project.clj b/project.clj index dc2bf72..ec8389a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.11.0-SNAPSHOT" +(defproject com.taoensso/nippy "2.11.0-alpha1" :author "Peter Taoussanis " :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy"