From a005a9d7fa2e56ea15abdadbe162ef00c0988100 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 1 Jun 2015 11:00:32 +0700 Subject: [PATCH 01/38] Switch to dynamic default-freeze-compressor-selector --- src/taoensso/nippy.clj | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index c0f01ec..30c3cb3 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -391,11 +391,14 @@ (> ba-len 128) lz4-compressor :else nil))) -(encore/defonce* default-freeze-compressor-selector_ - "EXPERIMENTAL. - Determines the global default default compressor selector - (fn [^bytes ba])->compressor used by `(freeze {:compressor :auto <...>})." - (atom default-freeze-compressor-selector)) +(encore/defonce* ^:dynamic ^:private *default-freeze-compressor-selector* + default-freeze-compressor-selector) + +(defn set-default-freeze-compressor-selector! + "Sets global dynamic (fn selector [^bytes ba])->compressor used by + `(freeze {:compressor :auto <...>})." + [selector] + (alter-var-root #'*default-freeze-compressor-selector* (constantly selector))) (defn freeze "Serializes arg (any Clojure data type) to a byte array. To freeze custom @@ -417,7 +420,7 @@ (if (identical? compressor :auto) (if skip-header? lz4-compressor - (@default-freeze-compressor-selector_ ba)) + (*default-freeze-compressor-selector* ba)) (if (fn? compressor) (compressor ba) ; Assume compressor selector fn compressor ; Assume compressor From 82294f54f93db7495f10bf1868341fcc43b274bb Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 1 Jun 2015 11:07:50 +0700 Subject: [PATCH 02/38] Switch to dynamic custom-readers --- src/taoensso/nippy.clj | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 30c3cb3..e7cd446 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -466,9 +466,9 @@ (def ^:private class-method-sig (into-array Class [IPersistentMap])) -(declare ^:private custom-readers) +(declare ^:dynamic *custom-readers*) (defn- read-custom! [type-id in] - (if-let [custom-reader (get @custom-readers type-id)] + (if-let [custom-reader (get *custom-readers* type-id)] (try (custom-reader in) (catch Exception e @@ -756,7 +756,9 @@ (.writeShort ~out ~(coerce-custom-type-id custom-type-id)))) ~@body))) -(defonce custom-readers (atom {})) ; { (fn [data-input]) ...} +(def ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) +(defn swap-custom-readers! [f] (alter-var-root #'*custom-readers* f)) + (defmacro extend-thaw "Extends Nippy to support thawing of a custom type with given id: (extend-thaw :foo/my-type [data-input] ; Keyword id @@ -766,13 +768,15 @@ (->MyType (.readUTF data-input)))" [custom-type-id [in] & body] (assert-custom-type-id custom-type-id) - (when (contains? @custom-readers (coerce-custom-type-id custom-type-id)) + (when (contains? *custom-readers* (coerce-custom-type-id custom-type-id)) (println (format "Warning: resetting Nippy thaw for custom type with id: %s" custom-type-id))) - `(swap! custom-readers assoc - ~(coerce-custom-type-id custom-type-id) - (fn [~(with-meta in {:tag 'java.io.DataInput})] - ~@body))) + `(swap-custom-readers! + (fn [m#] + (assoc m# + ~(coerce-custom-type-id custom-type-id) + (fn [~(with-meta in {:tag 'java.io.DataInput})] + ~@body))))) (comment (defrecord MyType [data]) (extend-freeze MyType 1 [x out] (.writeUTF out (:data x))) From 5a623870fe475c37a12e03e917de2b45c492e6a2 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 1 Jun 2015 11:09:59 +0700 Subject: [PATCH 03/38] Warn about thaw-id reset at runtime rather than expansion time --- src/taoensso/nippy.clj | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index e7cd446..5744831 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -768,20 +768,22 @@ (->MyType (.readUTF data-input)))" [custom-type-id [in] & body] (assert-custom-type-id custom-type-id) - (when (contains? *custom-readers* (coerce-custom-type-id custom-type-id)) - (println (format "Warning: resetting Nippy thaw for custom type with id: %s" - custom-type-id))) - `(swap-custom-readers! - (fn [m#] - (assoc m# - ~(coerce-custom-type-id custom-type-id) - (fn [~(with-meta in {:tag 'java.io.DataInput})] - ~@body))))) + `(do + (when (contains? *custom-readers* ~(coerce-custom-type-id custom-type-id)) + (println (format "Warning: resetting Nippy thaw for custom type with id: %s" + ~custom-type-id))) + (swap-custom-readers! + (fn [m#] + (assoc m# + ~(coerce-custom-type-id custom-type-id) + (fn [~(with-meta in {:tag 'java.io.DataInput})] + ~@body)))))) -(comment (defrecord MyType [data]) - (extend-freeze MyType 1 [x out] (.writeUTF out (:data x))) - (extend-thaw 1 [in] (->MyType (.readUTF in))) - (thaw (freeze (->MyType "Joe")))) +(comment + (defrecord MyType [data]) + (extend-freeze MyType 1 [x out] (.writeUTF out (:data x))) + (extend-thaw 1 [in] (->MyType (.readUTF in))) + (thaw (freeze (->MyType "Joe")))) ;;; Some useful custom types - EXPERIMENTAL From e5ccd258486d0d70e578960e66fe463bf7a0f327 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 1 Jun 2015 11:53:55 +0700 Subject: [PATCH 04/38] Fix custom-readers var declaration --- src/taoensso/nippy.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 5744831..adde63b 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -466,7 +466,9 @@ (def ^:private class-method-sig (into-array Class [IPersistentMap])) -(declare ^:dynamic *custom-readers*) +(def ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) +(defn swap-custom-readers! [f] (alter-var-root #'*custom-readers* f)) + (defn- read-custom! [type-id in] (if-let [custom-reader (get *custom-readers* type-id)] (try @@ -756,9 +758,6 @@ (.writeShort ~out ~(coerce-custom-type-id custom-type-id)))) ~@body))) -(def ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) -(defn swap-custom-readers! [f] (alter-var-root #'*custom-readers* f)) - (defmacro extend-thaw "Extends Nippy to support thawing of a custom type with given id: (extend-thaw :foo/my-type [data-input] ; Keyword id @@ -780,6 +779,7 @@ ~@body)))))) (comment + *custom-readers* (defrecord MyType [data]) (extend-freeze MyType 1 [x out] (.writeUTF out (:data x))) (extend-thaw 1 [in] (->MyType (.readUTF in))) From 12d90a05f8d5719e07b624476dd7c1253425c3a4 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 2 Jun 2015 12:45:57 +0700 Subject: [PATCH 05/38] Bump dev Clojure version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5cfb136..4773eab 100644 --- a/project.clj +++ b/project.clj @@ -24,7 +24,7 @@ :server-jvm {:jvm-opts ^:replace ["-server" "-Xms1024m" "-Xmx2048m"]} :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} - :1.7 {:dependencies [[org.clojure/clojure "1.7.0-beta1"]]} + :1.7 {:dependencies [[org.clojure/clojure "1.7.0-RC1"]]} :test {:jvm-opts ["-Xms1024m" "-Xmx2048m"] :dependencies [[expectations "2.1.1"] [org.clojure/test.check "0.7.0"] From 11545690c8b07a84f82d5cadcf16ff69ac343c0a Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 28 Jun 2015 13:47:58 +0700 Subject: [PATCH 06/38] NB switch to encore edn reader/writer --- project.clj | 2 +- src/taoensso/nippy.clj | 17 ++++++++--------- src/taoensso/nippy/benchmarks.clj | 9 ++++----- src/taoensso/nippy/utils.clj | 7 +++---- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/project.clj b/project.clj index 4773eab..5e1d1bd 100644 --- a/project.clj +++ b/project.clj @@ -14,7 +14,7 @@ :dependencies [[org.clojure/clojure "1.4.0"] [org.clojure/tools.reader "0.9.2"] - [com.taoensso/encore "1.32.0"] + [com.taoensso/encore "1.38.0"] [org.iq80.snappy/snappy "0.3"] [org.tukaani/xz "1.5"] [net.jpountz.lz4/lz4 "1.3"]] diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index adde63b..a9e8765 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -2,8 +2,7 @@ "High-performance JVM Clojure serialization library. Originally adapted from Deep-Freeze." {:author "Peter Taoussanis"} - (:require [clojure.tools.reader.edn :as edn] - [taoensso.encore :as encore] + (:require [taoensso.encore :as encore] [taoensso.nippy (utils :as utils) (compression :as compression) @@ -21,7 +20,7 @@ ;;;; Encore version check -(let [min-encore-version 1.28] ; For `backport-run!` support +(let [min-encore-version 1.38] (if-let [assert! (ns-resolve 'taoensso.encore 'assert-min-encore-version)] (assert! min-encore-version) (throw @@ -79,7 +78,7 @@ (def ^:const id-bytes (int 2)) (def ^:const id-nil (int 3)) (def ^:const id-boolean (int 4)) - (def ^:const id-reader (int 5)) ; Fallback #2: pr-str output + (def ^:const id-reader (int 5)) ; Fallback #2 (def ^:const id-serializable (int 6)) ; Fallback #1 (def ^:const id-char (int 10)) @@ -325,7 +324,7 @@ (def ^:dynamic *final-freeze-fallback* "Alpha - subject to change." nil) (defn freeze-fallback-as-str "Alpha-subject to change." [x out] - (freeze-to-out* {:nippy/unfreezable (pr-str x) :type (type x)} out)) + (freeze-to-out* {:nippy/unfreezable (encore/pr-edn x) :type (type x)} out)) (comment (require '[clojure.core.async :as async]) @@ -349,13 +348,13 @@ (do (when-debug-mode (println (format "DEBUG - Reader fallback: %s" (type x)))) (write-id out id-reader) - (write-utf8 out (pr-str x))) + (write-utf8 out (encore/pr-edn x))) :else ; Fallback #3: *final-freeze-fallback* (if-let [ffb *final-freeze-fallback*] (ffb x out) (throw (ex-info (format "Unfreezable type: %s %s" (type x) (str x)) {:type (type x) - :as-str (pr-str x)})))))) + :as-str (encore/pr-edn x)})))))) (def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta)) (def ^:private get-head-ba @@ -496,7 +495,7 @@ id-reader (let [edn (read-utf8 in)] (try - (edn/read-string {:readers *data-readers*} edn) + (encore/read-edn {:readers *data-readers*} edn) (catch Exception e {:type :reader :throwable e @@ -581,7 +580,7 @@ id-uuid (UUID. (.readLong in) (.readLong in)) ;;; DEPRECATED - id-old-reader (edn/read-string (.readUTF in)) + id-old-reader (encore/read-edn (.readUTF in)) id-old-string (.readUTF in) id-old-map (apply hash-map (encore/repeatedly-into* [] (* 2 (.readInt in)) (thaw-from-in in))) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index a48ff76..fe379e6 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,7 +1,6 @@ (ns taoensso.nippy.benchmarks {:author "Peter Taoussanis"} - (:require [clojure.tools.reader.edn :as edn] - [clojure.data.fressian :as fressian] + (:require [clojure.data.fressian :as fressian] [taoensso.encore :as encore] [taoensso.nippy :as nippy :refer (freeze thaw)])) @@ -37,8 +36,8 @@ (println (str "\nLap " (inc l) "/" laps "...")) (when reader? ; Slow - (println {:reader (bench1 #(pr-str %) #(edn/read-string %) - #(count (.getBytes ^String % "UTF-8")))})) + (println {:reader (bench1 encore/pr-edn encore/read-edn + #(count (.getBytes ^String % "UTF-8")))})) (println {:default (bench1 #(freeze % {}) #(thaw % {}))}) @@ -59,7 +58,7 @@ (println "\nDone! (Time for cake?)") true) -(comment (edn/read-string (pr-str data)) +(comment (encore/read-edn (encore/pr-edn data)) (bench1 fressian-freeze fressian-thaw)) (comment diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index a5b8ca6..aa265ea 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -1,8 +1,7 @@ (ns taoensso.nippy.utils {:author "Peter Taoussanis"} - (:require [clojure.string :as str] - [clojure.tools.reader.edn :as edn] - [taoensso.encore :as encore]) + (:require [clojure.string :as str] + [taoensso.encore :as encore]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable ObjectOutputStream ObjectInputStream])) @@ -34,7 +33,7 @@ (cast class object) true))))) -(def readable? (memoize-type-test (fn [x] (-> x pr-str (edn/read-string)) true))) +(def readable? (memoize-type-test (fn [x] (-> x encore/pr-edn encore/read-edn) true))) (comment (serializable? "Hello world") From 6b4e1341ee41945fea153ac91ea9f5439bda9f83 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 28 Jun 2015 13:50:27 +0700 Subject: [PATCH 07/38] Realign type ids --- src/taoensso/nippy.clj | 98 +++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index a9e8765..8911bf2 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -73,69 +73,69 @@ ;; ** Negative ids reserved for user-defined types ** ;; - (def ^:const id-reserved (int 0)) - ;; 1 - (def ^:const id-bytes (int 2)) - (def ^:const id-nil (int 3)) - (def ^:const id-boolean (int 4)) - (def ^:const id-reader (int 5)) ; Fallback #2 - (def ^:const id-serializable (int 6)) ; Fallback #1 + (def ^:const id-reserved (int 0)) + ;; 1 ; Deprecated + (def ^:const id-bytes (int 2)) + (def ^:const id-nil (int 3)) + (def ^:const id-boolean (int 4)) + (def ^:const id-reader (int 5)) ; Fallback #2 + (def ^:const id-serializable (int 6)) ; Fallback #1 - (def ^:const id-char (int 10)) - ;; 11 - ;; 12 - (def ^:const id-string (int 13)) - (def ^:const id-keyword (int 14)) + (def ^:const id-char (int 10)) + ;; 11 ; Deprecated + ;; 12 ; Deprecated + (def ^:const id-string (int 13)) + (def ^:const id-keyword (int 14)) - (def ^:const id-list (int 20)) - (def ^:const id-vector (int 21)) - ;; 22 - (def ^:const id-set (int 23)) - (def ^:const id-seq (int 24)) - (def ^:const id-meta (int 25)) - (def ^:const id-queue (int 26)) - (def ^:const id-map (int 27)) - (def ^:const id-sorted-set (int 28)) - (def ^:const id-sorted-map (int 29)) + (def ^:const id-list (int 20)) + (def ^:const id-vector (int 21)) + ;; 22 ; Deprecated + (def ^:const id-set (int 23)) + (def ^:const id-seq (int 24)) + (def ^:const id-meta (int 25)) + (def ^:const id-queue (int 26)) + (def ^:const id-map (int 27)) + (def ^:const id-sorted-set (int 28)) + (def ^:const id-sorted-map (int 29)) - (def ^:const id-byte (int 40)) - (def ^:const id-short (int 41)) - (def ^:const id-integer (int 42)) - (def ^:const id-long (int 43)) - (def ^:const id-bigint (int 44)) - (def ^:const id-biginteger (int 45)) + (def ^:const id-byte (int 40)) + (def ^:const id-short (int 41)) + (def ^:const id-integer (int 42)) + (def ^:const id-long (int 43)) + (def ^:const id-bigint (int 44)) + (def ^:const id-biginteger (int 45)) - (def ^:const id-float (int 60)) - (def ^:const id-double (int 61)) - (def ^:const id-bigdec (int 62)) + (def ^:const id-float (int 60)) + (def ^:const id-double (int 61)) + (def ^:const id-bigdec (int 62)) - (def ^:const id-ratio (int 70)) + (def ^:const id-ratio (int 70)) - (def ^:const id-record (int 80)) - ;; (def ^:const id-type (int 81)) ; TODO? + (def ^:const id-record (int 80)) + ;; (def ^:const id-type (int 81)) ; TODO? (def ^:const id-prefixed-custom (int 82)) - (def ^:const id-date (int 90)) - (def ^:const id-uuid (int 91)) + (def ^:const id-date (int 90)) + (def ^:const id-uuid (int 91)) ;;; Optimized, common-case types (v2.6+) - (def ^:const id-byte-as-long (int 100)) ; 1 vs 8 bytes - (def ^:const id-short-as-long (int 101)) ; 2 vs 8 bytes - (def ^:const id-int-as-long (int 102)) ; 4 vs 8 bytes - ;; (def ^:const id-compact-long (int 103)) ; 6->7 vs 8 bytes + (def ^:const id-byte-as-long (int 100)) ; 1 vs 8 bytes + (def ^:const id-short-as-long (int 101)) ; 2 vs 8 bytes + (def ^:const id-int-as-long (int 102)) ; 4 vs 8 bytes + ;; (def ^:const id-compact-long (int 103)) ; 6->7 vs 8 bytes ;; - (def ^:const id-string-small (int 105)) ; 1 vs 4 byte length prefix - (def ^:const id-keyword-small (int 106)) ; '' + (def ^:const id-string-small (int 105)) ; 1 vs 4 byte length prefix + (def ^:const id-keyword-small (int 106)) ; '' ;; - ;; (def ^:const id-vector-small (int 110)) ; '' - ;; (def ^:const id-set-small (int 111)) ; '' - ;; (def ^:const id-map-small (int 112)) ; '' + ;; (def ^:const id-vector-small (int 110)) ; '' + ;; (def ^:const id-set-small (int 111)) ; '' + ;; (def ^:const id-map-small (int 112)) ; '' ;;; DEPRECATED (old types will be supported only for thawing) - (def ^:const id-old-reader (int 1)) ; as of 0.9.2, for +64k support - (def ^:const id-old-string (int 11)) ; as of 0.9.2, for +64k support - (def ^:const id-old-map (int 22)) ; as of 0.9.0, for more efficient thaw - (def ^:const id-old-keyword (int 12)) ; as of 2.0.0-alpha5, for str consistecy + (def ^:const id-old-reader (int 1)) ; v0.9.2+ for +64k support + (def ^:const id-old-string (int 11)) ; v0.9.2+ for +64k support + (def ^:const id-old-map (int 22)) ; v0.9.0+ for more efficient thaw + (def ^:const id-old-keyword (int 12)) ; v2.0.0-alpha5+ for str consistecy ) ;;;; Ns imports (mostly for convenience of lib consumers) From da671cbba4b981927d8d96f4fd5632201975adbb Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 8 Aug 2015 00:08:45 +0700 Subject: [PATCH 08/38] Housekeeping --- src/taoensso/nippy.clj | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 8911bf2..648094e 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -208,7 +208,6 @@ (println (format "DEBUG - freezer-coll: %s for %s" ~type (type ~'x))))) (if (counted? ~'x) (do (.writeInt ~'out (count ~'x)) - ;; (doseq [i# ~'x] (freeze-to-out ~'out i#)) (encore/backport-run! (fn [i#] (freeze-to-out ~'out i#)) ~'x)) (let [bas# (ByteArrayOutputStream.) sout# (DataOutputStream. bas#) @@ -223,9 +222,6 @@ (defmacro ^:private freezer-kvs [type id & body] `(freezer ~type ~id (.writeInt ~'out (* 2 (count ~'x))) - ;; (doseq [kv# ~'x] - ;; (freeze-to-out ~'out (key kv#)) - ;; (freeze-to-out ~'out (val kv#))) (encore/backport-run! (fn [kv#] (freeze-to-out ~'out (key kv#)) @@ -916,8 +912,5 @@ ;;;; Deprecated API -(def freeze-to-stream! "DEPRECATED: Use `freeze-to-out!` instead." - freeze-to-out!) - -(def thaw-from-stream! "DEPRECATED: Use `thaw-from-in!` instead." - thaw-from-in!) +(def freeze-to-stream! "DEPRECATED: Use `freeze-to-out!` instead." freeze-to-out!) +(def thaw-from-stream! "DEPRECATED: Use `thaw-from-in!` instead." thaw-from-in!) From d89649deebea8eec63c63404014cd247319c0b10 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 8 Aug 2015 00:13:59 +0700 Subject: [PATCH 09/38] Bump deps --- project.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index 5e1d1bd..ddb8dbe 100644 --- a/project.clj +++ b/project.clj @@ -14,8 +14,8 @@ :dependencies [[org.clojure/clojure "1.4.0"] [org.clojure/tools.reader "0.9.2"] - [com.taoensso/encore "1.38.0"] - [org.iq80.snappy/snappy "0.3"] + [com.taoensso/encore "1.38.0"] ; 2.x needs Clojure 1.5+ + [org.iq80.snappy/snappy "0.4"] [org.tukaani/xz "1.5"] [net.jpountz.lz4/lz4 "1.3"]] @@ -27,9 +27,9 @@ :1.7 {:dependencies [[org.clojure/clojure "1.7.0-RC1"]]} :test {:jvm-opts ["-Xms1024m" "-Xmx2048m"] :dependencies [[expectations "2.1.1"] - [org.clojure/test.check "0.7.0"] + [org.clojure/test.check "0.8.2"] ;; [com.cemerick/double-check "0.6.1"] - [org.clojure/data.fressian "0.2.0"] + [org.clojure/data.fressian "0.2.1"] [org.xerial.snappy/snappy-java "1.1.1.7"]]} :dev [:1.7 :test {:plugins From 56b33e23f05a689c020aa4309e58b0ae07a151e1 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 14 Sep 2015 12:52:35 +0700 Subject: [PATCH 10/38] Perf: fix boxed math on long compression --- src/taoensso/nippy.clj | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 648094e..2be93b1 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -279,24 +279,25 @@ (write-utf8 out (.getName (class x))) ; Reflect (freeze-to-out out (into {} x))) -(freezer Byte id-byte (.writeByte out x)) -(freezer Short id-short (.writeShort out x)) -(freezer Integer id-integer (.writeInt out x)) -;;(freezer Long id-long (.writeLong out x)) +(freezer Byte id-byte (.writeByte out x)) +(freezer Short id-short (.writeShort out x)) +(freezer Integer id-integer (.writeInt out x)) +;;(freezer Long id-long (.writeLong out x)) (extend-type Long ; Optimized common-case type Freezable (freeze-to-out* [x ^DataOutput out] - (cond - (<= Byte/MIN_VALUE x Byte/MAX_VALUE) - (do (write-id out id-byte-as-long) (.writeByte out x)) + (let [^long x x] + (cond + (and (<= Byte/MIN_VALUE x) (<= x Byte/MAX_VALUE)) + (do (write-id out id-byte-as-long) (.writeByte out x)) - (<= Short/MIN_VALUE x Short/MAX_VALUE) - (do (write-id out id-short-as-long) (.writeShort out x)) + (and (<= Short/MIN_VALUE x) (<= x Short/MAX_VALUE)) + (do (write-id out id-short-as-long) (.writeShort out x)) - (<= Integer/MIN_VALUE x Integer/MAX_VALUE) - (do (write-id out id-int-as-long) (.writeInt out x)) + (and (<= Integer/MIN_VALUE x) (<= x Integer/MAX_VALUE)) + (do (write-id out id-int-as-long) (.writeInt out x)) - :else (do (write-id out id-long) (.writeLong out x))))) + :else (do (write-id out id-long) (.writeLong out x)))))) ;; @@ -361,6 +362,7 @@ (defn- wrap-header [data-ba head-meta] (if-let [head-ba (get-head-ba head-meta)] + ;; TODO Would be nice if we could avoid the array copy here: (encore/ba-concat head-ba data-ba) (throw (ex-info (format "Unrecognized header meta: %s" head-meta) {:head-meta head-meta})))) From 1ba3c38ab2284aa2f674657fdb115832808f6e3c Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 14 Sep 2015 14:08:06 +0700 Subject: [PATCH 11/38] Update benchmarks --- src/taoensso/nippy/benchmarks.clj | 6 ++++++ test/taoensso/nippy/tests/main.clj | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index fe379e6..427d33f 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -64,6 +64,12 @@ (comment ;; (bench {:reader? true :lzma2? true :fressian? true :laps 3}) ;; (bench {:laps 4}) + ;; (bench {:laps 1}) + + ;;; 2015 Sep 14 - v2.10.0-beta1, Clojure 1.7.0 + {:default {:round 6870, :freeze 4376, :thaw 2494, :size 16227}} + {:fast {:round 6104, :freeze 3743, :thaw 2361, :size 17013}} + {:encrypted {:round 12155, :freeze 6908, :thaw 5247, :size 16244}} ;;; 2015 June 4 - v2.9.0, Clojure 1.7.0-RC1 {:reader {:round 155353, :freeze 44192, :thaw 111161, :size 27693}} diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 3422822..5781f52 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -164,4 +164,4 @@ ;;;; Benchmarks -;; (expect (benchmarks/bench {})) ; Also tests :cached passwords +(expect (benchmarks/bench {})) ; Also tests :cached passwords From 3f9fe327e0eafc6f5580234a1bfa7d160ea29ea4 Mon Sep 17 00:00:00 2001 From: Karsten Schmidt Date: Tue, 15 Sep 2015 15:16:55 +0100 Subject: [PATCH 12/38] [#70] move small? check outside write-bytes macro body, replace if-not's w/ if's (@postspectacular) --- src/taoensso/nippy.clj | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 2be93b1..bd0a9bd 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -161,14 +161,14 @@ (defmacro write-id [out id] `(.writeByte ~out ~id)) (defmacro write-bytes [out ba & [small?]] - (let [out (with-meta out {:tag 'java.io.DataOutput}) - ba (with-meta ba {:tag 'bytes})] + (let [out (with-meta out {:tag 'java.io.DataOutput}) + ba (with-meta ba {:tag 'bytes}) + ;; Optimization, must be known before id's written + ;; `byte` to throw on range error + [wc wr] (if small? [byte 'writeByte] [int 'writeInt])] `(let [out# ~out, ba# ~ba size# (alength ba#)] - (if ~small? ; Optimization, must be known before id's written - (.writeByte out# (byte size#)) ; `byte` to throw on range error - (.writeInt out# (int size#)) ; `int` '' - ) + (. out# ~wr (~wc size#)) (.write out# ba# 0 size#)))) (defmacro write-biginteger [out x] @@ -405,7 +405,7 @@ encryptor aes128-encryptor} :as opts}]] (let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze - compressor (if-not legacy-mode? compressor snappy-compressor) + compressor (if legacy-mode? snappy-compressor compressor) encryptor (when password (if-not legacy-mode? encryptor nil)) skip-header? (or skip-header? legacy-mode?) baos (ByteArrayOutputStream.) @@ -423,8 +423,8 @@ compressor ; Assume compressor )) - ba (if-not compressor ba (compress compressor ba)) - ba (if-not encryptor ba (encrypt encryptor password ba))] + ba (if compressor (compress compressor ba) ba) + ba (if encryptor (encrypt encryptor password ba) ba)] (if skip-header? ba (wrap-header ba @@ -653,18 +653,20 @@ e))) thaw-data (fn [data-ba compressor-id encryptor-id] - (let [compressor (if-not (identical? compressor :auto) compressor - (get-auto-compressor compressor-id)) - encryptor (if-not (identical? encryptor :auto) encryptor - (get-auto-encryptor encryptor-id))] + (let [compressor (if (identical? compressor :auto) + (get-auto-compressor compressor-id) + compressor) + encryptor (if (identical? encryptor :auto) + (get-auto-encryptor encryptor-id) + encryptor)] (when (and encryptor (not password)) (ex "Password required for decryption.")) (try (let [ba data-ba - ba (if-not encryptor ba (decrypt encryptor password ba)) - ba (if-not compressor ba (decompress compressor ba)) + ba (if encryptor (decrypt encryptor password ba) ba) + ba (if compressor (decompress compressor ba) ba) dis (DataInputStream. (ByteArrayInputStream. ba))] (thaw-from-in! dis)) @@ -791,9 +793,9 @@ ba-len (alength ba) compress? (> ba-len 1024)] (.writeBoolean out compress?) - (if-not compress? (write-bytes out ba) - (let [ba* (compress lzma2-compressor ba)] - (write-bytes out ba*))))) + (if compress? + (write-bytes out (compress lzma2-compressor ba)) + (write-bytes out ba)))) (extend-thaw 128 [in] (let [compressed? (.readBoolean in) From e403c1741750fc90ae2e304e7388d4605f9434b2 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 16 Sep 2015 00:45:33 +0700 Subject: [PATCH 13/38] Housekeeping for 0a35b8c --- src/taoensso/nippy.clj | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index bd0a9bd..f15a4d9 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -161,15 +161,17 @@ (defmacro write-id [out id] `(.writeByte ~out ~id)) (defmacro write-bytes [out ba & [small?]] - (let [out (with-meta out {:tag 'java.io.DataOutput}) - ba (with-meta ba {:tag 'bytes}) - ;; Optimization, must be known before id's written - ;; `byte` to throw on range error - [wc wr] (if small? [byte 'writeByte] [int 'writeInt])] - `(let [out# ~out, ba# ~ba - size# (alength ba#)] - (. out# ~wr (~wc size#)) - (.write out# ba# 0 size#)))) + (let [out (with-meta out {:tag 'java.io.DataOutput}) + ba (with-meta ba {:tag 'bytes})] + (if small? ; Optimization, must be known before id's written + `(let [out# ~out, ba# ~ba + size# (alength ba#)] + (.writeByte out# (byte size#)) + (.write out# ba# 0 size#)) + `(let [out# ~out, ba# ~ba + size# (alength ba#)] + (.writeInt out# (int size#)) + (.write out# ba# 0 size#))))) (defmacro write-biginteger [out x] (let [x (with-meta x {:tag 'java.math.BigInteger})] @@ -423,8 +425,8 @@ compressor ; Assume compressor )) - ba (if compressor (compress compressor ba) ba) - ba (if encryptor (encrypt encryptor password ba) ba)] + ba (if compressor (compress compressor ba) ba) + ba (if encryptor (encrypt encryptor password ba) ba)] (if skip-header? ba (wrap-header ba @@ -665,8 +667,8 @@ (try (let [ba data-ba - ba (if encryptor (decrypt encryptor password ba) ba) - ba (if compressor (decompress compressor ba) ba) + ba (if encryptor (decrypt encryptor password ba) ba) + ba (if compressor (decompress compressor ba) ba) dis (DataInputStream. (ByteArrayInputStream. ba))] (thaw-from-in! dis)) From c0fcedf72e0c793c8ade670973c392b54c7d751d Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 16 Sep 2015 00:49:00 +0700 Subject: [PATCH 14/38] Use Encore v2+ for dev (benching) lein profile --- project.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index ddb8dbe..6bef8d9 100644 --- a/project.clj +++ b/project.clj @@ -24,7 +24,7 @@ :server-jvm {:jvm-opts ^:replace ["-server" "-Xms1024m" "-Xmx2048m"]} :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} - :1.7 {:dependencies [[org.clojure/clojure "1.7.0-RC1"]]} + :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} :test {:jvm-opts ["-Xms1024m" "-Xmx2048m"] :dependencies [[expectations "2.1.1"] [org.clojure/test.check "0.8.2"] @@ -32,7 +32,9 @@ [org.clojure/data.fressian "0.2.1"] [org.xerial.snappy/snappy-java "1.1.1.7"]]} :dev [:1.7 :test - {:plugins + {:dependencies + [[com.taoensso/encore "2.13.0"]] + :plugins [[lein-pprint "1.1.1"] [lein-ancient "0.6.7"] [lein-expectations "0.0.8"] From d57788125a7733b3fdb2a8cd1449f46036833f9d Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 16 Sep 2015 01:04:45 +0700 Subject: [PATCH 15/38] Update benchmarks --- src/taoensso/nippy/benchmarks.clj | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 427d33f..f1a4351 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -66,7 +66,15 @@ ;; (bench {:laps 4}) ;; (bench {:laps 1}) - ;;; 2015 Sep 14 - v2.10.0-beta1, Clojure 1.7.0 + ;;; 2015 Sep 15 - v2.10.0-alpha6, Clojure 1.7.0 + {:reader {:round 94901, :freeze 25781, :thaw 69120, :size 27686}} + {:lzma2 {:round 65127, :freeze 43150, :thaw 21977, :size 11244}} + {:encrypted {:round 12590, :freeze 7565, :thaw 5025, :size 16148}} + {:fressian {:round 12085, :freeze 9168, :thaw 2917, :size 16972}} + {:default {:round 6974, :freeze 4582, :thaw 2392, :size 16123}} + {:fast {:round 6255, :freeze 3724, :thaw 2531, :size 17013}} + + ;;; 2015 Sep 14 - v2.10.0-alpha5, Clojure 1.7.0-RC1 {:default {:round 6870, :freeze 4376, :thaw 2494, :size 16227}} {:fast {:round 6104, :freeze 3743, :thaw 2361, :size 17013}} {:encrypted {:round 12155, :freeze 6908, :thaw 5247, :size 16244}} From a3847a481892b6bfdce7a06ac80d84bf009e7788 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 17 Sep 2015 10:55:09 +0700 Subject: [PATCH 16/38] Misc housekeeping --- project.clj | 2 +- src/taoensso/nippy.clj | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/project.clj b/project.clj index 6bef8d9..82aed6e 100644 --- a/project.clj +++ b/project.clj @@ -33,7 +33,7 @@ [org.xerial.snappy/snappy-java "1.1.1.7"]]} :dev [:1.7 :test {:dependencies - [[com.taoensso/encore "2.13.0"]] + [[com.taoensso/encore "2.15.0"]] :plugins [[lein-pprint "1.1.1"] [lein-ancient "0.6.7"] diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index f15a4d9..6180f20 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -1,7 +1,7 @@ (ns taoensso.nippy "High-performance JVM Clojure serialization library. Originally adapted from - Deep-Freeze." - {:author "Peter Taoussanis"} + Deep-Freeze (https://goo.gl/OePPGr)." + {:author "Peter Taoussanis (@ptaoussanis)"} (:require [taoensso.encore :as encore] [taoensso.nippy (utils :as utils) @@ -18,17 +18,9 @@ PersistentQueue PersistentTreeMap PersistentTreeSet PersistentList ; LazySeq IRecord ISeq])) -;;;; Encore version check - -(let [min-encore-version 1.38] - (if-let [assert! (ns-resolve 'taoensso.encore 'assert-min-encore-version)] - (assert! min-encore-version) - (throw - (ex-info - (format - "Insufficient com.taoensso/encore version (< %s). You may have a Leiningen dependency conflict (see http://goo.gl/qBbLvC for solution)." - min-encore-version) - {:min-version min-encore-version})))) +(if (vector? taoensso.encore/encore-version) + (encore/assert-min-encore-version [1 38 0]) ; Note v1.x for Clojure 1.4 support + (encore/assert-min-encore-version 1.38)) ;;;; Nippy data format ;; * 4-byte header (Nippy v2.x+) (may be disabled but incl. by default) [1]. From 41d3dcd467b26adff76bfd3264a43fd8035d1616 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 17 Sep 2015 11:10:51 +0700 Subject: [PATCH 17/38] `*default-freeze-compressor-selector*` should be public --- src/taoensso/nippy.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 6180f20..58fdca0 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -382,12 +382,12 @@ (> ba-len 128) lz4-compressor :else nil))) -(encore/defonce* ^:dynamic ^:private *default-freeze-compressor-selector* +(encore/defonce* ^:dynamic *default-freeze-compressor-selector* + "(fn selector [^bytes ba])->compressor used by `(freeze {:compressor :auto})." default-freeze-compressor-selector) (defn set-default-freeze-compressor-selector! - "Sets global dynamic (fn selector [^bytes ba])->compressor used by - `(freeze {:compressor :auto <...>})." + "Sets root binding of `*default-freeze-compressor-selector*`." [selector] (alter-var-root #'*default-freeze-compressor-selector* (constantly selector))) From 40b39db9eb4afd29690e288b259523162768c9d1 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 14 Sep 2015 17:09:45 +0700 Subject: [PATCH 18/38] v2.10.0-beta1 --- CHANGELOG.md | 17 ++++++++++++++++- README.md | 3 ++- project.clj | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2325ab8..ce91620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ > This project uses [Break Versioning](https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md) as of **Aug 16, 2014**. +## v2.10.0-beta1 / 2015 Sep 17 + +> This is a major **non-breaking** feature/performance release + +* **DEPRECATION NOTICE**: this will be the last major release with support for Clojure 1.4 +* **Performance**: various small performance improvements +* **New**: dynamic `*default-freeze-compressor-selector*`, `set-default-freeze-compressor-selector!` util +* **New**: dynamic `*custom-readers*`, `swap-custom-readers!` util +* **New**: edn writes now override dynamic `*print-level*`, `*print-length*` for safety + +```clojure +[com.taoensso/nippy "2.10.0-beta1"] +``` + + ## v2.9.1 / 2015 Sep 14 > This is a hotfix release with an **important fix** for Nippy encryption users @@ -13,7 +28,7 @@ ## v2.9.0 / 2015 Jun 1 -> This is a major, **non-breaking** release that improves performance and makes thawing more resilient to certain failures. Identical to **v2.9.0-RC3**. +> This is a major **non-breaking** release that improves performance and makes thawing more resilient to certain failures. Identical to **v2.9.0-RC3**. * **Robustness**: improve error handling for unthawable records * **Performance**: switch `doseq` -> (faster) `run!` calls diff --git a/README.md b/README.md index 28b9142..7a31edb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ **[API docs][]** | **[CHANGELOG][]** | [other Clojure libs][] | [Twitter][] | [contact/contrib](#contact--contributing) | current [Break Version][]: ```clojure -[com.taoensso/nippy "2.9.1"] ; Stable, see CHANGELOG for details +[com.taoensso/nippy "2.9.1"] ; Stable +[com.taoensso/nippy "2.10.0-beta1"] ; Dev, see CHANGELOG for details ``` # Nippy, a Clojure serialization library diff --git a/project.clj b/project.clj index 82aed6e..770590b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.9.1" +(defproject com.taoensso/nippy "2.10.0-beta1" :author "Peter Taoussanis " :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" From 9d4db3106ead9735e870e4f2650dde932324d291 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 17 Sep 2015 14:05:00 +0700 Subject: [PATCH 19/38] Perf: create Ratio's directly --- src/taoensso/nippy.clj | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 58fdca0..5bb251d 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -565,8 +565,12 @@ id-double (.readDouble in) id-bigdec (BigDecimal. (read-biginteger in) (.readInt in)) - id-ratio (/ (bigint (read-biginteger in)) - (bigint (read-biginteger in))) + ;; id-ratio (/ (bigint (read-biginteger in)) + ;; (bigint (read-biginteger in))) + + id-ratio (clojure.lang.Ratio. + (read-biginteger in) + (read-biginteger in)) id-date (Date. (.readLong in)) id-uuid (UUID. (.readLong in) (.readLong in)) From aa9a18088fa168bd6f0442e56ba8b9e4b0e39049 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 17 Sep 2015 13:40:24 +0700 Subject: [PATCH 20/38] Experimental/perf: kvs work directly against MapEntry --- src/taoensso/nippy.clj | 43 +++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 5bb251d..f321c01 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -86,7 +86,7 @@ (def ^:const id-seq (int 24)) (def ^:const id-meta (int 25)) (def ^:const id-queue (int 26)) - (def ^:const id-map (int 27)) + (def ^:const id-map (int 30)) (def ^:const id-sorted-set (int 28)) (def ^:const id-sorted-map (int 29)) @@ -124,10 +124,11 @@ ;; (def ^:const id-map-small (int 112)) ; '' ;;; DEPRECATED (old types will be supported only for thawing) - (def ^:const id-old-reader (int 1)) ; v0.9.2+ for +64k support - (def ^:const id-old-string (int 11)) ; v0.9.2+ for +64k support - (def ^:const id-old-map (int 22)) ; v0.9.0+ for more efficient thaw - (def ^:const id-old-keyword (int 12)) ; v2.0.0-alpha5+ for str consistecy + (def ^:const id-reader-v1 (int 1)) ; v0.9.2+ for +64k support + (def ^:const id-string-v1 (int 11)) ; v0.9.2+ for +64k support + (def ^:const id-map-v1 (int 22)) ; v0.9.0+ for more efficient thaw + (def ^:const id-keyword-v1 (int 12)) ; v2.0.0-alpha5+ for str consistecy + (def ^:const id-map-v2 (int 27)) ; v2.11 for more efficient freeze+thaw ) ;;;; Ns imports (mostly for convenience of lib consumers) @@ -215,13 +216,22 @@ (defmacro ^:private freezer-kvs [type id & body] `(freezer ~type ~id - (.writeInt ~'out (* 2 (count ~'x))) + (.writeInt ~'out (* 2 (count ~'x))) ; The *2 is vestigial (encore/backport-run! (fn [kv#] (freeze-to-out ~'out (key kv#)) (freeze-to-out ~'out (val kv#))) ~'x))) +(defmacro ^:private freezer-mapent-kvs [type id & body] + `(freezer ~type ~id + (.writeInt ~'out (count ~'x)) + (encore/backport-run! + (fn [kv#] + (freeze-to-out ~'out (.getKey ^clojure.lang.MapEntry kv#)) + (freeze-to-out ~'out (.getValue ^clojure.lang.MapEntry kv#))) + ~'x))) + (freezer (Class/forName "[B") id-bytes (write-bytes out ^bytes x)) (freezer nil id-nil) (freezer Boolean id-boolean (.writeBoolean out x)) @@ -259,7 +269,7 @@ (freezer-coll PersistentTreeSet id-sorted-set) (freezer-kvs PersistentTreeMap id-sorted-map) -(freezer-kvs APersistentMap id-map) +(freezer-mapent-kvs APersistentMap id-map) (freezer-coll APersistentVector id-vector) (freezer-coll APersistentSet id-set) (freezer-coll PersistentList id-list) ; No APersistentList @@ -452,9 +462,15 @@ (defmacro ^:private read-kvs [in coll] `(let [in# ~in] - (encore/repeatedly-into* ~coll (quot (.readInt in#) 2) + (encore/repeatedly-into* ~coll + (quot (.readInt in#) 2) ; The /2 is vestigial [(thaw-from-in in#) (thaw-from-in in#)]))) +(defmacro ^:private read-mapent-kvs [in coll] + `(let [in# ~in] + (encore/repeatedly-into* ~coll (.readInt in#) + (clojure.lang.MapEntry. (thaw-from-in in#) (thaw-from-in in#))))) + (def ^:private class-method-sig (into-array Class [IPersistentMap])) (def ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) @@ -540,7 +556,7 @@ id-list (into '() (rseq (read-coll in []))) id-vector (read-coll in []) id-set (read-coll in #{}) - id-map (read-kvs in {}) + id-map (read-mapent-kvs in {}) id-seq (or (seq (read-coll in [])) (lazy-seq nil) ; Empty coll ) @@ -576,11 +592,12 @@ id-uuid (UUID. (.readLong in) (.readLong in)) ;;; DEPRECATED - id-old-reader (encore/read-edn (.readUTF in)) - id-old-string (.readUTF in) - id-old-map (apply hash-map (encore/repeatedly-into* [] + id-map-v2 (read-kvs in {}) + id-reader-v1 (encore/read-edn (.readUTF in)) + id-string-v1 (.readUTF in) + id-map-v1 (apply hash-map (encore/repeatedly-into* [] (* 2 (.readInt in)) (thaw-from-in in))) - id-old-keyword (keyword (.readUTF in)) + id-keyword-v1 (keyword (.readUTF in)) id-prefixed-custom ; Prefixed custom type (let [hash-id (.readShort in)] From 36abe07f2b7c516395af18c008c3552e8100b64e Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 17 Sep 2015 18:48:03 +0700 Subject: [PATCH 21/38] Revert "Experimental/perf: kvs work directly against MapEntry" This reverts commit e150775cfe82f8206ddc88034417421e200851fa. --- src/taoensso/nippy.clj | 43 +++++++++++++----------------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index f321c01..5bb251d 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -86,7 +86,7 @@ (def ^:const id-seq (int 24)) (def ^:const id-meta (int 25)) (def ^:const id-queue (int 26)) - (def ^:const id-map (int 30)) + (def ^:const id-map (int 27)) (def ^:const id-sorted-set (int 28)) (def ^:const id-sorted-map (int 29)) @@ -124,11 +124,10 @@ ;; (def ^:const id-map-small (int 112)) ; '' ;;; DEPRECATED (old types will be supported only for thawing) - (def ^:const id-reader-v1 (int 1)) ; v0.9.2+ for +64k support - (def ^:const id-string-v1 (int 11)) ; v0.9.2+ for +64k support - (def ^:const id-map-v1 (int 22)) ; v0.9.0+ for more efficient thaw - (def ^:const id-keyword-v1 (int 12)) ; v2.0.0-alpha5+ for str consistecy - (def ^:const id-map-v2 (int 27)) ; v2.11 for more efficient freeze+thaw + (def ^:const id-old-reader (int 1)) ; v0.9.2+ for +64k support + (def ^:const id-old-string (int 11)) ; v0.9.2+ for +64k support + (def ^:const id-old-map (int 22)) ; v0.9.0+ for more efficient thaw + (def ^:const id-old-keyword (int 12)) ; v2.0.0-alpha5+ for str consistecy ) ;;;; Ns imports (mostly for convenience of lib consumers) @@ -216,22 +215,13 @@ (defmacro ^:private freezer-kvs [type id & body] `(freezer ~type ~id - (.writeInt ~'out (* 2 (count ~'x))) ; The *2 is vestigial + (.writeInt ~'out (* 2 (count ~'x))) (encore/backport-run! (fn [kv#] (freeze-to-out ~'out (key kv#)) (freeze-to-out ~'out (val kv#))) ~'x))) -(defmacro ^:private freezer-mapent-kvs [type id & body] - `(freezer ~type ~id - (.writeInt ~'out (count ~'x)) - (encore/backport-run! - (fn [kv#] - (freeze-to-out ~'out (.getKey ^clojure.lang.MapEntry kv#)) - (freeze-to-out ~'out (.getValue ^clojure.lang.MapEntry kv#))) - ~'x))) - (freezer (Class/forName "[B") id-bytes (write-bytes out ^bytes x)) (freezer nil id-nil) (freezer Boolean id-boolean (.writeBoolean out x)) @@ -269,7 +259,7 @@ (freezer-coll PersistentTreeSet id-sorted-set) (freezer-kvs PersistentTreeMap id-sorted-map) -(freezer-mapent-kvs APersistentMap id-map) +(freezer-kvs APersistentMap id-map) (freezer-coll APersistentVector id-vector) (freezer-coll APersistentSet id-set) (freezer-coll PersistentList id-list) ; No APersistentList @@ -462,15 +452,9 @@ (defmacro ^:private read-kvs [in coll] `(let [in# ~in] - (encore/repeatedly-into* ~coll - (quot (.readInt in#) 2) ; The /2 is vestigial + (encore/repeatedly-into* ~coll (quot (.readInt in#) 2) [(thaw-from-in in#) (thaw-from-in in#)]))) -(defmacro ^:private read-mapent-kvs [in coll] - `(let [in# ~in] - (encore/repeatedly-into* ~coll (.readInt in#) - (clojure.lang.MapEntry. (thaw-from-in in#) (thaw-from-in in#))))) - (def ^:private class-method-sig (into-array Class [IPersistentMap])) (def ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) @@ -556,7 +540,7 @@ id-list (into '() (rseq (read-coll in []))) id-vector (read-coll in []) id-set (read-coll in #{}) - id-map (read-mapent-kvs in {}) + id-map (read-kvs in {}) id-seq (or (seq (read-coll in [])) (lazy-seq nil) ; Empty coll ) @@ -592,12 +576,11 @@ id-uuid (UUID. (.readLong in) (.readLong in)) ;;; DEPRECATED - id-map-v2 (read-kvs in {}) - id-reader-v1 (encore/read-edn (.readUTF in)) - id-string-v1 (.readUTF in) - id-map-v1 (apply hash-map (encore/repeatedly-into* [] + id-old-reader (encore/read-edn (.readUTF in)) + id-old-string (.readUTF in) + id-old-map (apply hash-map (encore/repeatedly-into* [] (* 2 (.readInt in)) (thaw-from-in in))) - id-keyword-v1 (keyword (.readUTF in)) + id-old-keyword (keyword (.readUTF in)) id-prefixed-custom ; Prefixed custom type (let [hash-id (.readShort in)] From cea505484a08d839b7a2df5d530986c3d57ac937 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 17 Sep 2015 18:51:29 +0700 Subject: [PATCH 22/38] Note re double vestigial kvs length --- src/taoensso/nippy.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 5bb251d..9ee0bfb 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -215,7 +215,7 @@ (defmacro ^:private freezer-kvs [type id & body] `(freezer ~type ~id - (.writeInt ~'out (* 2 (count ~'x))) + (.writeInt ~'out (* 2 (count ~'x))) ; *2 here is vestigial (encore/backport-run! (fn [kv#] (freeze-to-out ~'out (key kv#)) @@ -452,7 +452,7 @@ (defmacro ^:private read-kvs [in coll] `(let [in# ~in] - (encore/repeatedly-into* ~coll (quot (.readInt in#) 2) + (encore/repeatedly-into* ~coll (quot (.readInt in#) 2) ; /2 here is vestigial [(thaw-from-in in#) (thaw-from-in in#)]))) (def ^:private class-method-sig (into-array Class [IPersistentMap])) From db375e7686291b8bc2c2fdd487dec61f21354ba7 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 19 Sep 2015 11:04:33 +0700 Subject: [PATCH 23/38] Perf: anon fn is faster here --- src/taoensso/nippy.clj | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 9ee0bfb..04aee44 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -448,12 +448,13 @@ (defmacro read-compact-long [in] `(long (BigInteger. (read-bytes ~in :small)))) (defmacro ^:private read-coll [in coll] - `(let [in# ~in] (encore/repeatedly-into* ~coll (.readInt in#) (thaw-from-in in#)))) + `(let [in# ~in] (encore/repeatedly-into ~coll (.readInt in#) + (fn [] (thaw-from-in in#))))) (defmacro ^:private read-kvs [in coll] `(let [in# ~in] - (encore/repeatedly-into* ~coll (quot (.readInt in#) 2) ; /2 here is vestigial - [(thaw-from-in in#) (thaw-from-in in#)]))) + (encore/repeatedly-into ~coll (quot (.readInt in#) 2) ; /2 here is vestigial + (fn [] [(thaw-from-in in#) (thaw-from-in in#)])))) (def ^:private class-method-sig (into-array Class [IPersistentMap])) @@ -578,8 +579,8 @@ ;;; DEPRECATED id-old-reader (encore/read-edn (.readUTF in)) id-old-string (.readUTF in) - id-old-map (apply hash-map (encore/repeatedly-into* [] - (* 2 (.readInt in)) (thaw-from-in in))) + id-old-map (apply hash-map (encore/repeatedly-into [] (* 2 (.readInt in)) + (fn [] (thaw-from-in in)))) id-old-keyword (keyword (.readUTF in)) id-prefixed-custom ; Prefixed custom type From ce39987b8cb1057968a86e5cea4b02ad47c7a130 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 26 Sep 2015 10:41:14 +0700 Subject: [PATCH 24/38] Bump deps --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 770590b..55ad7f5 100644 --- a/project.clj +++ b/project.clj @@ -30,10 +30,10 @@ [org.clojure/test.check "0.8.2"] ;; [com.cemerick/double-check "0.6.1"] [org.clojure/data.fressian "0.2.1"] - [org.xerial.snappy/snappy-java "1.1.1.7"]]} + [org.xerial.snappy/snappy-java "1.1.2"]]} :dev [:1.7 :test {:dependencies - [[com.taoensso/encore "2.15.0"]] + [[com.taoensso/encore "2.18.0"]] :plugins [[lein-pprint "1.1.1"] [lein-ancient "0.6.7"] From b20321622b75cef78818424f2490ceb2d13da196 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 26 Sep 2015 10:42:36 +0700 Subject: [PATCH 25/38] v2.10.0-RC1 --- CHANGELOG.md | 4 ++-- README.md | 2 +- project.clj | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce91620..389e8a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ > This project uses [Break Versioning](https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md) as of **Aug 16, 2014**. -## v2.10.0-beta1 / 2015 Sep 17 +## v2.10.0-RC1 / 2015 Sep 26 > This is a major **non-breaking** feature/performance release @@ -11,7 +11,7 @@ * **New**: edn writes now override dynamic `*print-level*`, `*print-length*` for safety ```clojure -[com.taoensso/nippy "2.10.0-beta1"] +[com.taoensso/nippy "2.10.0-RC1"] ``` diff --git a/README.md b/README.md index 7a31edb..5c7a81b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ```clojure [com.taoensso/nippy "2.9.1"] ; Stable -[com.taoensso/nippy "2.10.0-beta1"] ; Dev, see CHANGELOG for details +[com.taoensso/nippy "2.10.0-RC1"] ; Dev, see CHANGELOG for details ``` # Nippy, a Clojure serialization library diff --git a/project.clj b/project.clj index 55ad7f5..360469c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.10.0-beta1" +(defproject com.taoensso/nippy "2.10.0-RC1" :author "Peter Taoussanis " :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" From 67d1dfd34fb1fed7915b1ff220c7880c451921fb Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 26 Sep 2015 10:45:24 +0700 Subject: [PATCH 26/38] README typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c7a81b..3b64c2f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ **[API docs][]** | **[CHANGELOG][]** | [other Clojure libs][] | [Twitter][] | [contact/contrib](#contact--contributing) | current [Break Version][]: ```clojure -[com.taoensso/nippy "2.9.1"] ; Stable +[com.taoensso/nippy "2.9.1"] ; Stable [com.taoensso/nippy "2.10.0-RC1"] ; Dev, see CHANGELOG for details ``` From f1af76635abddd9aad51047f925e01a242c59f2e Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 14:28:47 +0700 Subject: [PATCH 27/38] Project.clj housekeeping, drop support for Clojure 1.4 Clojure 1.4 support is becoming more and more hassle; not worth it --- project.clj | 13 +++++-------- test/taoensso/nippy/tests/main.clj | 2 ++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/project.clj b/project.clj index 360469c..b4aa224 100644 --- a/project.clj +++ b/project.clj @@ -12,9 +12,9 @@ *unchecked-math* :warn-on-boxed} :dependencies - [[org.clojure/clojure "1.4.0"] + [[org.clojure/clojure "1.5.1"] [org.clojure/tools.reader "0.9.2"] - [com.taoensso/encore "1.38.0"] ; 2.x needs Clojure 1.5+ + [com.taoensso/encore "2.18.0"] [org.iq80.snappy/snappy "0.4"] [org.tukaani/xz "1.5"] [net.jpountz.lz4/lz4 "1.3"]] @@ -25,16 +25,14 @@ :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} + :1.8 {:dependencies [[org.clojure/clojure "1.8.0-alpha5"]]} :test {:jvm-opts ["-Xms1024m" "-Xmx2048m"] :dependencies [[expectations "2.1.1"] [org.clojure/test.check "0.8.2"] - ;; [com.cemerick/double-check "0.6.1"] [org.clojure/data.fressian "0.2.1"] [org.xerial.snappy/snappy-java "1.1.2"]]} :dev [:1.7 :test - {:dependencies - [[com.taoensso/encore "2.18.0"]] - :plugins + {:plugins [[lein-pprint "1.1.1"] [lein-ancient "0.6.7"] [lein-expectations "0.0.8"] @@ -44,8 +42,7 @@ :test-paths ["test" "src"] :aliases - {"test-all" ["with-profile" "default:+1.5:+1.6:+1.7" "expectations"] - ;; "test-all" ["with-profile" "default:+1.6" "expectations"] + {"test-all" ["with-profile" "+1.5:+1.6:+1.7:+1.8" "expectations"] "test-auto" ["with-profile" "+test" "autoexpect"] "deploy-lib" ["do" "deploy" "clojars," "install"] "start-dev" ["with-profile" "+server-jvm" "repl" ":headless"]} diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 5781f52..24c2c55 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -14,6 +14,8 @@ ;;;; Core +(expect (do (println (str "Clojure version: " *clojure-version*)) true)) + (expect test-data ((comp thaw freeze) test-data)) (expect test-data ((comp #(thaw % {}) #(freeze % {:legacy-mode true})) From bbbc12ce309244bc0009ec75cc49cecaacba6ee0 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 10:17:07 +0700 Subject: [PATCH 28/38] Misc hk --- src/taoensso/nippy.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 04aee44..7ae27fb 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -527,12 +527,12 @@ id-boolean (.readBoolean in) id-char (.readChar in) - id-string (read-utf8 in) + id-string (read-utf8 in) id-keyword (keyword (read-utf8 in)) ;;; Optimized, common-case types (v2.6+) - id-string-small (String. (read-bytes in :small) "UTF-8") - id-keyword-small (keyword (String. (read-bytes in :small) "UTF-8")) + id-string-small (read-utf8 in :small) + id-keyword-small (keyword (read-utf8 in :small)) id-queue (read-coll in (PersistentQueue/EMPTY)) id-sorted-set (read-coll in (sorted-set)) From 2f27666d055b59049403623654c4e2aef5e21c92 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 10:23:56 +0700 Subject: [PATCH 29/38] Rename deprecated type ids Making room for >1 deprecated id per type --- src/taoensso/nippy.clj | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 7ae27fb..afd80f3 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -124,10 +124,10 @@ ;; (def ^:const id-map-small (int 112)) ; '' ;;; DEPRECATED (old types will be supported only for thawing) - (def ^:const id-old-reader (int 1)) ; v0.9.2+ for +64k support - (def ^:const id-old-string (int 11)) ; v0.9.2+ for +64k support - (def ^:const id-old-map (int 22)) ; v0.9.0+ for more efficient thaw - (def ^:const id-old-keyword (int 12)) ; v2.0.0-alpha5+ for str consistecy + (def ^:const id-reader-depr1 (int 1)) ; v0.9.2+ for +64k support + (def ^:const id-string-depr1 (int 11)) ; v0.9.2+ for +64k support + (def ^:const id-map-depr1 (int 22)) ; v0.9.0+ for more efficient thaw + (def ^:const id-keyword-depr1 (int 12)) ; v2.0.0-alpha5+ for str consistecy ) ;;;; Ns imports (mostly for convenience of lib consumers) @@ -577,11 +577,11 @@ id-uuid (UUID. (.readLong in) (.readLong in)) ;;; DEPRECATED - id-old-reader (encore/read-edn (.readUTF in)) - id-old-string (.readUTF in) - id-old-map (apply hash-map (encore/repeatedly-into [] (* 2 (.readInt in)) + id-reader-depr1 (encore/read-edn (.readUTF in)) + id-string-depr1 (.readUTF in) + id-map-depr1 (apply hash-map (encore/repeatedly-into [] (* 2 (.readInt in)) (fn [] (thaw-from-in in)))) - id-old-keyword (keyword (.readUTF in)) + id-keyword-depr1 (keyword (.readUTF in)) id-prefixed-custom ; Prefixed custom type (let [hash-id (.readShort in)] From 53d993e132282d3fab8b8af8e9942873b48e71e5 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 15:02:06 +0700 Subject: [PATCH 30/38] Tests housekeeping --- test/taoensso/nippy/tests/main.clj | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 24c2c55..d060abf 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -40,7 +40,7 @@ (check-props/for-all [val check-gen/any] (= val (thaw (freeze val))))))) -;;; These can sometimes crash the JVM +;;; Trying to decrypt random (invalid) data can actually crash JVM ;; (expect Exception (thaw (freeze test-data {:password "malformed"}))) ;; (expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) ;; (expect Exception (thaw (freeze test-data {:password [:salted "p"]}) @@ -77,25 +77,26 @@ (nippy/extend-thaw :nippy-tests/MyType [s] (->MyType (.readUTF s))) (let [type (->MyType "val")] (= type (thaw (freeze type)))))) -;;;; Stable binary representation of vals ; EXPERIMENTAL +;;;; Stable binary representation of vals (expect (seq (freeze test-data)) (seq (freeze test-data))) ; f(x)=f(y) | x=y -;;; As above, but try multiple times (catch protocol interface races): +;; As above, but try multiple times to catch possible protocol interface races: (expect #(every? true? %) (repeatedly 1000 (fn [] (= (seq (freeze test-data)) (seq (freeze test-data)))))) -(expect (seq (-> test-data freeze)) ; f(x)=f(f-1(f(x))) - (seq (-> test-data freeze thaw freeze))) - -;;; As above, but with repeated refreeze (catch protocol interface races): -(expect (= (seq (freeze test-data)) - (seq (reduce (fn [frozen _] (freeze (thaw frozen))) - (freeze test-data) (range 1000))))) - -;;; +;; NB abandoning - no way to do this reliably w/o appropriate contracts from +;; (seq ): +;; +;; (expect (seq (-> test-data freeze)) ; f(x)=f(f-1(f(x))) +;; (seq (-> test-data freeze thaw freeze))) +;; +;; As above, but with repeated refreeze to catch possible protocol interface races: +;; (expect (= (seq (freeze test-data)) +;; (seq (reduce (fn [frozen _] (freeze (thaw frozen))) +;; (freeze test-data) (range 1000))))) (defn qc-prop-bijection [& [n]] (let [bin->val (atom {}) @@ -125,7 +126,6 @@ (let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)] [result (vals bin->val)])) -;; (expect #(:result %) (qc-prop-bijection 120)) ; Time is n-non-linear (expect #(:result %) (qc-prop-bijection 80)) ;;;; Thread safety From 7ae954a22985905b5114b91e70155101b388e11d Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 26 Sep 2015 11:31:49 +0700 Subject: [PATCH 31/38] Micro optimization: remove & args --- src/taoensso/nippy.clj | 189 ++++++++++++++++++++--------------------- 1 file changed, 93 insertions(+), 96 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index afd80f3..92f4809 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -366,7 +366,7 @@ (defn freeze-to-out! "Low-level API. Serializes arg (any Clojure data type) to a DataOutput." - [^DataOutput data-output x & _] + [^DataOutput data-output x] (freeze-to-out data-output x)) (defn default-freeze-compressor-selector @@ -394,40 +394,41 @@ (defn freeze "Serializes arg (any Clojure data type) to a byte array. To freeze custom types, extend the Clojure reader or see `extend-freeze`." - ^bytes [x & [{:keys [compressor encryptor password skip-header?] - :or {compressor :auto - encryptor aes128-encryptor} - :as opts}]] - (let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze - compressor (if legacy-mode? snappy-compressor compressor) - encryptor (when password (if-not legacy-mode? encryptor nil)) - skip-header? (or skip-header? legacy-mode?) - baos (ByteArrayOutputStream.) - dos (DataOutputStream. baos)] - (freeze-to-out! dos x) - (let [ba (.toByteArray baos) + (^bytes [x] (freeze x nil)) + (^bytes [x {:keys [compressor encryptor password skip-header?] + :or {compressor :auto + encryptor aes128-encryptor} + :as opts}] + (let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze + compressor (if legacy-mode? snappy-compressor compressor) + encryptor (when password (if-not legacy-mode? encryptor nil)) + skip-header? (or skip-header? legacy-mode?) + baos (ByteArrayOutputStream.) + dos (DataOutputStream. baos)] + (freeze-to-out! dos x) + (let [ba (.toByteArray baos) - compressor - (if (identical? compressor :auto) - (if skip-header? - lz4-compressor - (*default-freeze-compressor-selector* ba)) - (if (fn? compressor) - (compressor ba) ; Assume compressor selector fn - compressor ; Assume compressor - )) + compressor + (if (identical? compressor :auto) + (if skip-header? + lz4-compressor + (*default-freeze-compressor-selector* ba)) + (if (fn? compressor) + (compressor ba) ; Assume compressor selector fn + compressor ; Assume compressor + )) - ba (if compressor (compress compressor ba) ba) - ba (if encryptor (encrypt encryptor password ba) ba)] + ba (if compressor (compress compressor ba) ba) + ba (if encryptor (encrypt encryptor password ba) ba)] - (if skip-header? ba - (wrap-header ba - {:compressor-id (when-let [c compressor] - (or (compression/standard-header-ids - (compression/header-id c)) :else)) - :encryptor-id (when-let [e encryptor] - (or (encryption/standard-header-ids - (encryption/header-id e)) :else))}))))) + (if skip-header? ba + (wrap-header ba + {:compressor-id (when-let [c compressor] + (or (compression/standard-header-ids + (compression/header-id c)) :else)) + :encryptor-id (when-let [e encryptor] + (or (encryption/standard-header-ids + (encryption/header-id e)) :else))})))))) ;;;; Thawing @@ -597,7 +598,7 @@ (defn thaw-from-in! "Low-level API. Deserializes a frozen object from given DataInput to its original Clojure data type." - [data-input & _] + [data-input] (thaw-from-in data-input)) (defn- try-parse-header [ba] @@ -634,73 +635,74 @@ Options include: :compressor - An ICompressor, :auto (requires Nippy header), or nil. :encryptor - An IEncryptor, :auto (requires Nippy header), or nil." - [^bytes ba - & [{:keys [compressor encryptor password v1-compatibility?] - :or {compressor :auto - encryptor :auto - v1-compatibility? true ; Recommend disabling when possible - } - :as opts}]] - (assert (not (contains? opts :headerless-meta)) - ":headerless-meta `thaw` option removed as of Nippy v2.7.") + ([ba] (thaw ba nil)) + ([^bytes ba + {:keys [v1-compatibility? compressor encryptor password] + :or {v1-compatibility? true ; Recommend disabling when possible + compressor :auto + encryptor :auto} + :as opts}] - (let [ex (fn [msg & [e]] (throw (ex-info (format "Thaw failed: %s" msg) - {:opts (merge opts - {:compressor compressor - :encryptor encryptor})} - e))) - thaw-data - (fn [data-ba compressor-id encryptor-id] - (let [compressor (if (identical? compressor :auto) - (get-auto-compressor compressor-id) - compressor) - encryptor (if (identical? encryptor :auto) - (get-auto-encryptor encryptor-id) - encryptor)] + (assert (not (:headerless-meta opts)) + ":headerless-meta `thaw` opt removed in Nippy v2.7+") - (when (and encryptor (not password)) - (ex "Password required for decryption.")) + (let [ex (fn [msg & [e]] (throw (ex-info (format "Thaw failed: %s" msg) + {:opts (merge opts + {:compressor compressor + :encryptor encryptor})} + e))) + thaw-data + (fn [data-ba compressor-id encryptor-id] + (let [compressor (if (identical? compressor :auto) + (get-auto-compressor compressor-id) + compressor) + encryptor (if (identical? encryptor :auto) + (get-auto-encryptor encryptor-id) + encryptor)] - (try - (let [ba data-ba - ba (if encryptor (decrypt encryptor password ba) ba) - ba (if compressor (decompress compressor ba) ba) - dis (DataInputStream. (ByteArrayInputStream. ba))] - (thaw-from-in! dis)) + (when (and encryptor (not password)) + (ex "Password required for decryption.")) - (catch Exception e - (ex "Decryption/decompression failure, or data unfrozen/damaged." - e))))) + (try + (let [ba data-ba + ba (if encryptor (decrypt encryptor password ba) ba) + ba (if compressor (decompress compressor ba) ba) + dis (DataInputStream. (ByteArrayInputStream. ba))] + (thaw-from-in! dis)) - ;; This is hackish and can actually currently result in JVM core dumps - ;; due to buggy Snappy behaviour, Ref. http://goo.gl/mh7Rpy. - thaw-nippy-v1-data - (fn [data-ba] - (if-not v1-compatibility? - (throw (Exception. "v1 compatibility disabled")) - (try (thaw-data data-ba :snappy nil) - (catch Exception _ - (thaw-data data-ba nil nil)))))] + (catch Exception e + (ex "Decryption/decompression failure, or data unfrozen/damaged." + e))))) - (if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?] - :as head-meta}] (try-parse-header ba)] - - ;; A well-formed header _appears_ to be present (it's possible though - ;; unlikely that this is a fluke and data is actually headerless): - (try (thaw-data data-ba compressor-id encryptor-id) - (catch Exception e - (try (thaw-nippy-v1-data data-ba) + ;; This is hackish and can actually currently result in JVM core dumps + ;; due to buggy Snappy behaviour, Ref. http://goo.gl/mh7Rpy. + thaw-nippy-v1-data + (fn [data-ba] + (if-not v1-compatibility? + (throw (Exception. "v1 compatibility disabled")) + (try (thaw-data data-ba :snappy nil) (catch Exception _ - (if unrecognized-meta? - (ex "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?" - e) - (throw e)))))) + (thaw-data data-ba nil nil)))))] - ;; Well-formed header definitely not present - (try (thaw-nippy-v1-data ba) - (catch Exception _ - (thaw-data ba :no-header :no-header)))))) + (if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?] + :as head-meta}] (try-parse-header ba)] + + ;; A well-formed header _appears_ to be present (it's possible though + ;; unlikely that this is a fluke and data is actually headerless): + (try (thaw-data data-ba compressor-id encryptor-id) + (catch Exception e + (try (thaw-nippy-v1-data data-ba) + (catch Exception _ + (if unrecognized-meta? + (ex "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?" + e) + (throw e)))))) + + ;; Well-formed header definitely not present + (try (thaw-nippy-v1-data ba) + (catch Exception _ + (thaw-data ba :no-header :no-header))))))) (comment (thaw (freeze "hello")) (thaw (freeze "hello" {:compressor nil})) @@ -912,8 +914,3 @@ (comment (inspect-ba (freeze "hello")) (seq (:data-ba (inspect-ba (freeze "hello"))))) - -;;;; Deprecated API - -(def freeze-to-stream! "DEPRECATED: Use `freeze-to-out!` instead." freeze-to-out!) -(def thaw-from-stream! "DEPRECATED: Use `thaw-from-in!` instead." thaw-from-in!) From 956ce7df7ec7ae8b2da5de3093173b40d79839e5 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 10:21:15 +0700 Subject: [PATCH 32/38] Micro optimization: `read-bytes` expansion --- src/taoensso/nippy.clj | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 92f4809..08dee69 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -435,12 +435,17 @@ (declare thaw-from-in) (defmacro read-bytes [in & [small?]] - `(let [in# ~in - size# (if ~small? ; Optimization, must be known before id's written - (.readByte in#) - (.readInt in#)) - ba# (byte-array size#)] - (.readFully in# ba# 0 size#) ba#)) + (if small? ; Optimization, must be known before id's written + `(let [in# ~in + size# (.readByte in#) + ba# (byte-array size#)] + (.readFully in# ba# 0 size#) + ba#) + `(let [in# ~in + size# (.readInt in#) + ba# (byte-array size#)] + (.readFully in# ba# 0 size#) + ba#))) (defmacro read-biginteger [in] `(BigInteger. (read-bytes ~in))) (defmacro read-utf8 [in & [small?]] From 1ae8e6c389be432b874cc8ebab109e01fb51aa15 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 28 Sep 2015 16:37:54 +0700 Subject: [PATCH 33/38] Micro optimization: destructure faster than explicit calls here --- src/taoensso/nippy.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 08dee69..ac6efd3 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -217,9 +217,9 @@ `(freezer ~type ~id (.writeInt ~'out (* 2 (count ~'x))) ; *2 here is vestigial (encore/backport-run! - (fn [kv#] - (freeze-to-out ~'out (key kv#)) - (freeze-to-out ~'out (val kv#))) + (fn [[k# v#]] + (freeze-to-out ~'out k#) + (freeze-to-out ~'out v#)) ~'x))) (freezer (Class/forName "[B") id-bytes (write-bytes out ^bytes x)) From ea9286dc901efcfd8829f855e9130fb77b87e73e Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 29 Sep 2015 09:33:56 +0700 Subject: [PATCH 34/38] Micro optimization: kv run is faster still --- src/taoensso/nippy.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index ac6efd3..ae64828 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -19,8 +19,8 @@ IRecord ISeq])) (if (vector? taoensso.encore/encore-version) - (encore/assert-min-encore-version [1 38 0]) ; Note v1.x for Clojure 1.4 support - (encore/assert-min-encore-version 1.38)) + (encore/assert-min-encore-version [2 16 0]) + (encore/assert-min-encore-version 2.16)) ;;;; Nippy data format ;; * 4-byte header (Nippy v2.x+) (may be disabled but incl. by default) [1]. @@ -202,7 +202,7 @@ (println (format "DEBUG - freezer-coll: %s for %s" ~type (type ~'x))))) (if (counted? ~'x) (do (.writeInt ~'out (count ~'x)) - (encore/backport-run! (fn [i#] (freeze-to-out ~'out i#)) ~'x)) + (encore/run!* (fn [i#] (freeze-to-out ~'out i#)) ~'x)) (let [bas# (ByteArrayOutputStream.) sout# (DataOutputStream. bas#) cnt# (reduce (fn [^long cnt# i#] @@ -216,8 +216,8 @@ (defmacro ^:private freezer-kvs [type id & body] `(freezer ~type ~id (.writeInt ~'out (* 2 (count ~'x))) ; *2 here is vestigial - (encore/backport-run! - (fn [[k# v#]] + (encore/run-kv! + (fn [k# v#] (freeze-to-out ~'out k#) (freeze-to-out ~'out v#)) ~'x))) From 9c33f4f5aca2a87e31d78dbb829f593bb45189c5 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sat, 26 Sep 2015 11:56:40 +0700 Subject: [PATCH 35/38] Update benchmarks --- benchmarks.png | Bin 18458 -> 17199 bytes src/taoensso/nippy/benchmarks.clj | 11 ++++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/benchmarks.png b/benchmarks.png index 9b4fc5d152ac7292ae44b1a3043dd7bae7173c23..9cd79829fb718923df996fd05742fc756401e9fa 100644 GIT binary patch literal 17199 zcmbunbyQqUvoAbIfFMDF1(y&!B)CHe1lQmm+}+)Ry9Rf+;1C!fxclJlHaLUBH{^NV z^L*>Q@44&Vd-flD_4MA|U0q%EtE#S^9U>y-aIxhf#b`@yx&+SAB+Un2E0|#3A z=l1*>;JB+LuPU5J@gj{!Zj68bs9;HSZ@_*Y2yn0Sz4luL(BEsrr-R>e=|Ew^&hoP0 zpnzZ)abSl`^@3^0IQE9{{HLA#J+ekygt`h{!w9j#y!u?OcdV|?UYg^w=t~N7E zUoo}va17^I?xW`@iS3Ya(w~V*0%u9)jZ< zTInOtA`SS}ttZuTZVagl)o7f<7d&kE<(F>hA|Cw+*BRXkSL9=)-e9J0?Io)@U^ z_Y+7@caF#_X;nZVbKQs-c=eu~cUVsi?BxWW=eycAjn~t(zY8%5|Bf6ye!#)0RarU6glTVar)bxNiwPyhXEAxNy(XS%2s*i(J~hZ6a<{mB|n5kPx_*dP^U zg6xjq7lt=u4pt5MS`CP5XjDgfmTt62FHOQeBU39K$C{z}clsvnrZx&Ub2!QrDlMVH zPBSi|8dut_2bz?JeC4HPmeXXzk%b^oWi7CWbz_>yI*-ChuCKwmPO>$|oABv8ha^*%A%3*e}iKzP$>bvBVumQKXjFqyqf2PfO@2ZM-ti zQ)5mZIxmf@Jnvdg7UZqWx4ln(_^rk49@tDYgw;h8N+4pJq?brW!)#@nM6pQGC@CLL zJAnO39{yYC2WMJ8W)X~B8fN${l+e{J`T@K}@MfO%3diwXy7k$zCzpWv@_Cwf23RAK zcg1I>{h;p9HrJp<~+O|6#xVSu)CD#-4#U z>3f4l8r6V7ycAC+NshEq1 zIA?$V`p=UB(B5d8&BpM~kzE;?jUXTc8_^$^Ix5rA+dz7wOs});?5Y{kW@ZV&8-Hge zc5*|8fdmihD|mmWYlLup;;)j+@P~jQk5l?ld`hG}v|<5Ouh zW86zM`0&&A0*eF z{QF+U66!)e@CY+_Ug+ zNAxwuYWhY(^790>h6=`P#CBsiV!OYc1pl(|m#q^^oEUfSl9AP$9S?%hVoU14AaZe4 zpgV?-$;kT?hHdyTw-e0=TI!K&k+u41%6z%lEIQ^zA02EjJ2~4(o>3AuCOm{5d)Yd? zHz%tSN)lVdgslosD9J)&3jgUujhYMzaPZ;HWC6gD1*m?q|2 z=7)gs9#74uQ*gBIO&gz~(_=q_*9KnOb+>+$Bks%Ms>vl?We-&8&zWGv@-M5A@Q>Wz z_Sbk`$)%wADZ#~)Q`7tT>7`zdP&6E{A=R0K3<4ywgpsY+QL7COs+|WWKvt7s?{i!~ zLJs9}7vf*LoEhzHT8+9fi`6-WiMiYqFQRWyIGW#0zkF**ih;hc7!gZUv@Bi;gJs%) z{Zh}E*Ks+-JxrgU_ZMAP%ZGleO=@kP$>i7cz{SWfIW;!v;BxfS9bfQ1ksY{OeUFhp z>uYCof^F!BHQetD>Q7tZRbsKue)f_Er|dyI1ZgA+1UG9^iKk&5k|X5WvV}E{Lbw*+ z3<)jPh_xR|q0(|CgDKd8MlKqYUxMa6^H_+b(v*FS(%?)`iu*?^$*##YKa=&*Jzy+6 za5XZCLNZAg2oM^r0NBpXE#C@i9c_T^Wlv*M9K9kdR&GYCt&0wYbi$Y)7PAAlw91!} zT0R*ZM9uBzr_YZ{w)<7q{B2EU$>=`Yw2ER4@J+ zaUm?OQ_Bsjro6irT*szRJ^4|?9FxL6w8OGoz^GGzPqE~OveC=w^RKG)V9)^!+ai%W zTfhSRwlyis8fwiZ&I@K4I~#MfchOAOxfRupKX}|kKdo+wu8aTf)NS)i*&HdX z9J9_yLT?l%E86nL&pD20Rvfu$@dtO;Av&E~(}c|lB~hs-{^g9jdku;m%Xiw-8t&08!P{H;{F6ZC|Tn zB3zl5Qw62(+Q_8J9?Loe$>t~BP}uB98FNt&E>_q3_`qRV$fIDfs0FJa>b zz1b5&n)$$w+nO3baCA}}Z|oHUC1UAL=WjE(*hYt;C{S5VDAzU^f2{eyZF9U*iRXBu z{%-%$UB@NkZ%Ke(e}v+k5+Ir?;Fwj63~}uMcKjCC=jx2Xw-Pf%00I~?%nT-s(JvqLwv9zs_BiA7)ICq4F6jBx7fDhVvR(RVi?zSA#yRF}e0Mbwur zEAbHqsnJ1wc7~!oVWC$~=~Bd}o>yL1C3J z(s7pQ_jqJip6o2lLhQ}SZAXptl$(bx4a$$cZb?x@Hz8lOx)dMsDR3??}^&X==Mo97J;+jTquGFP-i+VXwN;~PFz(BFdvAErv z6FRp|gpH<(?oiOS2V)GygSb*lI!@^cNBm6`UN?)4^9OD5jdL;Hhe zoJQ|##LX}I=DKM1?Nhd2Z|cm0`s(h|WvqDzV3Hmjl)qf_7S06ukC+yg_jYo~9;k%; z;fdInS>(^&aKe(gay3hWz1(uejz6Iw{fb%m z$c5<;QqcJBGY3+E(@alBMc{JnLh`j($XlOq3P3P}51K-JI5PYJ5|4tOD7K6{OJOcd zYWEOa^l&isMXB(g>+JxaD5Asp0Ac)0jW6QnunJ4>87`BOuGud_@9CH8yLQ{0%Pl0L zcY{bgS(T+pFgR25)01Y*Y|v2X+i`en!}zwQZijX)l1D7KRWs9N^eXuR0jAq$kba%Jr%vb51AcUW#34kbgf)AdQ<0x=D94QRB?Rne z6zGbMr{pH(ju%-^+Y8#vT~QsyVO0(Jz7hoMFeZiF6s>r3RkM1xSSo`Qy!Y-Z=ee<% zK?0w@h#`q|#T_sumTuiU@-EPs@7>Z;IA>8Y>a|Q9Br(;~C$NcF{iLVQ^}e$SijqjY zoWb-lK_jy-EY_-2=DeOnS-DcCd{_g|n#kW+Q8!KU91|IO*IvzG{Ly9k54@JIcIG^A z+PBoYYs-vonh-Z-2In5`KVoQSC`p^oGC0GM&2Y;7WU(goYH(t{zWH#~l+msH1<RF ze95n9uNJBC5FbYmVq-M>S%w+j)y9mG0r6&BGWI`_e7q*z0QO zyZUkJon8`<3(42~`_Ur97H_lTbai&(7k$dytplzAs|ICrLrHF*xzxI`l1ipX->ilG zz80^q@9bGglj#Wh&W0_YZ=>15(5&Z< z^WaYkmm;0m;jt~ob0qfVxfH5%7-uww=^8X3^XVKZ%Im0cPwG$nC0bmjsUvO6i0Vfs zOhc6cx%*qm<3AF$(NP6p2-fQkd3+u~l5>*I>cd4~X`{}U5~&+#uSCW8D#JJ$t$`ta zYgNB~hjSQg;rOBSBIjauWXKwb8DH8ByzJ&&sTOkOuw}BdUpi$6^^u>Zii1F|L(d!A=^~R_;Tdl?PKd%rh>`GGPAp?HM$P>^(Zvy$ueur z59%o*j0KsaGg!|3!xOBb>g$UsqAp!-6N^Ua3HLw$MdBX|%5(iCoBh7Q#qTm}=My6k zKO$7k_ez+TKJlz~6x-LDf&e#j0p=_}mOw@p8GN}^rsQSS{Hb`kV#gk8FZ9<}tevQh zvTa`quKWKBw`;XIm9!>nb4-WX1-IXEvWMC#&V+5d(2`RRe?>ed67IhDT7M4$5WN2jk-02hP(h$3S760r_$h-u^RV!LfHSsh<3R_mcERB-pg5;hHdytKg6LbI~2&L z6uE#0-e_;;rPZ_m+*liDa4d=4?CvvryoQGw*FbkyHq-)OTnN*tGCWx>Eg*p6XyqM_ zn*J&$uo{3{({D8t*{3)(et#+Q2zU9n_S8BN19~5tK!glBY_`&eqm|kb+7X0kxjChR z_}A5o_6~v^l`Wst<^-hA9xVAQVO?qx8_8=Uof8nFSyZ%{u4;n!@FZ89HG(oWVE4no zdVdV%Yk_$(h``C*a?-=&!A*7ar&?DyGUR`(tF9y&If;&zrcN-+&$4Gr@@ls@IgYSP&Ox-C~mPk(o~S%;=UUI9v0(^?6=rjcqUEKIo*BBc5C9PhztOYXCj%*Ykt*e8fr0vvi(7;4M8vY#`-u8i^qmDRQ5s9NPFW~X!aU9#a*zCPEE5R2odcYpxo#=a# zT%EGA(bT=Bmu!VeaVQu70|nj{9W_H+PcP>(ZMw_`^&mChsF#14oXiGn@`9Pv;oE75 z+`xu&gI+77n_~!H9U2bRZ&`} zj&F~duqW-aIqJ;6Pa4AiW%8DF9(H#N=8DZ44HbtvLiijWLhI4XhL4>9wW9VLAHrtk z>om)8%_hIpoi(ZS6_89^wl7T?g34UdJ9`m#@ussVY#3A0@CYXdN3N%e&3f9IG0xFD zb8)52v(^$dwMM~CPWG}qJuRx7wn0{kNPDT1@<^ zoaYy5ZB#1&Hr`$lQD}cL_zpD89UDE96Cq%gfzZCp@}m8Am{71B58WooPIU#q$om$r)&DB&+EW*($;->h%Mg!P zWOnUc%N#${^Mx^}oPQRXHZHkV#AbO`8g`Mtok0m<#IpM5_vGflO-w@nnKMO8q zJu{*qaU-ue<1`kJ9iIgoAT9CAy3l<@Fa7b!W*w%<0?K{ljmM82rlXJ}_9lvl-rFe2QygWrte54^?mgLYQLX16p4ZTV%Ye#JQFAUD--@?30$hfN##@zHszy z4JWBY+S+Jr*w|g3sj6r-yF1xUn9RXkK*CC8$lc@c8YSa{D$iyW5;k)~%g2r!-8Jfg zp%-q=0y>whzB~bf^Hx?686lIc86y&#h_G(3j|iAoP(~DifqN@w_(G0Rm=>iy{=88mO^AuINSIt3Dc);vI+IGqxB4n zL9HhCFQ;Q}^Sbpz((Co8%QUsZG8jVVbt+(p;6RO?Cu^q8=Be_90KjD6(%-m>>NM~0 zu6Kv=W%=Qq;bNg8aa+n`8L?BDn&^O-1X4EAX_id(j?2xOe z?3X)y=o*B)*yYph{6B5s1c%LGKU!CPqC$DoND{p;iA@v4NL)Vn1&k%AH3L5p-)SVy zm+GVeumgLrkGvm}=zU=~d~?VeMlGiZ7<5!z!XqU0!H^xT*2$MW#8GN*yZy0$Pv6Ra zupszs=Me+d_KBrW8&A%L_DX)){0gfdrU{Ot~%K8 z6+O2eXq}%#4ie^w<70)rvGzEsv-V~81q_A+?eQTUx$u6CLZh2JqTk+>zuSF@4({F( z|Ip*?lFr|oJlRSfZGIZbosq}R(PMOQZOi~NPm3)kqS73Zm=;%pNMWhAADlSCXW*}% z`I=1L%Wzrj=W4v~7EWW?u+&{wlMox3>Pmk!UTLa;i#96NPpd-(*Pp8FcLZ%STuFMp zVU(;{fhx|dKs@4NN_i=z8a!bd7;3#-W!Tr(Vr62Xfx%q)!CYH&xpQHC6qw7Oh4RZ?Q7c2L!!&;wCfMQkZ`Nh${sM%oIQKT3AiPg|&Mza8eE zq5s?D!aoUzf6@&9R%#b(BywweBIH6Jwd2pwYV7=BAF3nmrZeYUa=O+A<-^r;n&96W z{TcAz>U^G=_NVUON~>r1FOxoRbb1daU*0{K)wyZgzn}Ov#I+PhE!)%E%e0J$2u*kl zxL)6gjg+OF%X<o--IuAkv;P2anOCHk=Bo<7pPAit$cVxYaq8c?Q z$IDi-*p;p9<@sQclkNV|(Svj45lx^$a>>&`dh3a_@M7GOeBEr~@Xdl~>?q*?#;fP}N*X^%{mPC^7x@rb5%{t!xLp0c#vt<~0n zeq7wm{LGoA{Kd?4eQdlTKPV-+*l0ISwszkTn!s^OU^Zo3y8G6AdiEtI&v9oV%w8nJ z-Q9L={9=lu`RbQ{)0eH?dC*#2Z+f*xs_L#3!{+zjMmt$5qSlNJgyPAbxsvIm^bT<-?`B*m^;asc*)w-8&T{=(L)keBrAncA?*2gL zx_I#cyPrU>odur$p&5Fv>dT9br9FzR57zXTZu!?~OWAgJ;NsK=gPS#Al|-OZhSjmm zbZk;Z3lX$WphV1tij_>cfyv{;^15oq!AiQ>vUFfc`OQc}rKg+aH1ru&Z-v({PUV}l zB}(@zqm|uv=iaSc3p92^^Q7pNyiO&Of0A4A;+MB)`@?JJiJ=mFpW_C-&z0Hyqm+-A zA9SgcS5&pi3|uTLm;`>g)-epW$Q7&b8()bvd6`3wWYi^(nPfm`YBMX|4mIp8b}l7@ zP^ONRvi4N4?nk5*vGMSIlz;pB^-RT7L0eNbvuA0YVM*KbZd+Y| zWiTD0s?E+lT{aV@sRz~f9mXn^MA!Va=_x(q)|AJ29?4YywBhCL3Bs_Y+Sbd9pc)Yw zMW}AJKPYJ@m$=ggfmPdpzgq0T+XE4)_6zCTYi`ar3s(dEs9_xL#f%I?AlW%f9iXQ^ z)m`Y)0VD%F)Mz&o&2r0FmTWS`!7K_!%Ezk-2eg{XX#_|oi_-nlJ(;}!eY z&2_kE&=9e&n{n?}-UtFNqo1n{I~C51*r#hVRZj65UQMB}G|Z`} zwW*i4zOxVf{Jg#cDl;6*Zn5X5qg*v@Pb1b)ZEOF(N=*KnqMr~KJtPmoU73N z{`%CFxQ9qb?eX<(=HbHXn;#nst15^7sFY1_{{p_Rz#AQC%GhTH4!Yu5DsSlAQ6f}K zR+zpnTMImi2Fj=}dN$U2iyTaMdXy|$aS%*iu?L|vA4I@4IGq;*^QRw+KpW@KHGm4s zM=lnwn&aCclUKFxj9-D7RL&Q*)r@xYx_|d9CA@v8YfkkiqBT<~ncMSV?Tc9|zvKE@ z52DvWozp%pWb>>;dS1b;AGaOoJOYc(S3y@twXxF*J3&F*0v_%qeK8c&`0tOL+JJG2 zrTru5GwxgpsoMC`PhQs1pZ&}4mWb3>=7+O^+6|dQGVEs(P*H)RP^kug{Q z+5Wx3+_! zkZEoLj*D05l;hP+YudScx_@Aw);Qj|Jp@twnF~`E_fwI3vqB=5BDOzhuGso`OK$FA zO(OR464WC@wFiDX_X}9X!=WLoV9HoqeO;s?V^wy+;@vpi$Q@T^ZPBK6)0jCF2|5>n zcDJiU7ol!TCK@WiI5+4E6m_nTRir0f(m8q+JFsiwXk6H~IG3iCCZ-_JlKQLeV^zkG zY}A&ZKE#cG*UBOWi!eD_pK>_tKAb^=HcvaEMXN`ia&4`fzt1 z+d|pq*zBUmv@)(UvzvKwA`|qmLHl_kjjZ+}*u$MZInx67O-UP&HQ9lOu$fD)jXe&o zya}%+}Yh#!g@KHB*-b~*Fngmo0aBXwriLaot{5pxY+;_5(F$)4Syxtq5v z;L=X=gl!@EeuGlAzvS8eSC3{PN76=U6KHLIFbt^8!&E9|apC5g?PXatqcQX`0b*8} zjsnFm!OF93@2$W5reW7V%kQyKPR&L7l!iyr#*q;Q#p;u*|Lu8-|09>@J>qk3EC*M2 zATzc~`acE$%Ef<)Np%6GypveOQkFUs+wpu@-iTvDwg_th*I*(?W(tw>iwI_}(Lr0ELx$Q7QYD1;)E zt{#~#H+VpKkoX5*a)(2P##7t5_QCn!t!TxJF-cMY++J|mzEPu;t%lj)-SYVb84pLOdoM#9y`dE}mtfB9$)jkE_Q6lL)K11Z( z&)1~R0|dBaNt877#GN|4(ayU@bIAdj`Jaq4kOyE0A(01+J(@kDI7!$QPR_@OSga?X zrZXeGChAi`5rB)1XrV$bn=)ZlcaOkw5^i!doVoV%6mPxa02nF%*x)Z#<2lWZnBD(X zVeYKar+2{9t+6$`@y3UB$LA!cp0sLgudk08khT%H$X!hzv-WBMWv2h+x7qrfOO}W6 zecA?xE~sBk-wJhx=r2+*gsxObjS(~6SLqenmt>&s1Wu{|_Zj^Sz$oK?41KtX_B*i{oBsS?V_h-XPGj0|uHu z_P$uLud*#8L_NQ+k$5brgrnMwQnSJ6SSTM{$i;3$O}Xod|(C?&;>-ij=hm93@SyJ|gn`c;GpWvCldKNsW^kv`dL;rH~hNJ%X98rFSvc4dQ z)$mXjV#oWNjuTFz&<#!3auS|>3#)~#8fTR z_@ge++mepwJFiC3ePck*2P8nAe0XUb@U}4|==Acg)PNK1Jxs9IS-b0d!1@P!>_ExVjTpD%tJh29E zB{W!6!Z9Fr5q$8gLL9caxER%=b?B7gA35=0i-1Y~2p9}O$$Cp+vqXdeUlkzDPRf{W zq}2{rKWOTZa8J}Yp0qinp+0TP{1WUX2l4az{t~D7*PVsgZPSv=f&0rDbI$5lw>=D> z2QNc2DD}$Ki$CzDw!abzhauug2xRQE!!!Ntz`om9;=>cKZlr_BiP#RG4zr?&KQOIV z%!K!w71g-)_1&w(r?QI@z`w-cyj`{zpXXMkXywL-(?oK$^r{B6nwSwc%*Fc3{``HO=Ttn(9!r1t{|mawr(x8(p=OV2@6qm$yY>3OkM z5q_L%naxz~VX=Avyf6ez@c&x!VHMJCZsUv{>36Re4#n%@-i+B%JHqTy1p6MPB6Q;6 z!?S5*fjQC_EL>rUp@K9 z!}tR?ls@Le(Bh9-G}QJ)Oqyq*ZqtH?X{asdT98G;7J}RJayi^t_WA3|%T>LEcLyRL zhJAVl2OPbc`+X$xQU7816;>-uv`FjgH3@Mmhs#fuTBQKqS4M)LOI2)wl@PT~dAxDU ziGc$@duLCLOt7S3@E^D%e?qd%J$Df0C#Fjq$*26csNFd)c3MAP+2-5d{@$aV)u#gB z`1-PFhIr6-kXPmGT{1(My*F29TVJChzHNe8iHgmBhXYN*hXxM6aZ)cN75$|c&tOVC zWc=ZRyW%C&9V6;akzg)M1XLz|i@KTV1C{nmmY^9NJscMw z21(R^;(!oQzNTXlATCPHExnrHTtrdap zr+-jXoIyD7f#!<7#8Wfh)Zrr`g5Z{GX?N3YoXk$Bp_6SbG~*bhTM;p*TFdeKdS@)Kf&qOF*sT;Y zyKX0(azH}7`l1b{$v4d@Sl9a|5W!+ ztm%_qf_^+Ub*fe#AFIIq#5Y42euX;7CE8^y3T4%wKa4&f9sVcS{vQR_wKLFA^#7rU zE6BlYbJS{m6^qY?4#C8-$O56`oc5%HsuZS1?7c-TF`eCqHp3Aar?xs>d$CScSF z!pSV`i8l%0>3zWiK*87xD{S;U^t?fTbO1UM$+cL=kOvPOBZtWOgA6u_@1%EYFFQsw zWBHFvPY0Tl-D{yL7*E0AW>zj%)>}dzhqvA~4hFTu-keQjDums0wsWi3zrPbWu%6Xy zfZ2@=Q_sThU7F|TzEfae*?H`p^vMnOW#8MjhjsLlX%n~=YX`7K5<^gK)@%vY7bg)# zJHFZ&*sXD{>VI&n4G^;`cd=1KVCKH*L?b+z$cg%NuMluQ<6>}7Qdyw5auWW{AW%ui zqiP@XjH0}tE{i_hop?Hcu8T@Z+Q1)VCV^w*K*{ovh}TU*`z;Ro{1h6{aX;Lm<}qGX zj*ZW_b~KtkU<#!;3wq!p4N)7LAK`T3L=tnNTUr{2*~lFu(cUbJ=f!1gP}J|;aV#t= zCPhU%OeV%#{4ORDqz9587g}ty$Z?6(oQdRtN^?Ib}=3ACG(;c+I^tgqrXW|nJ;r+65os}3aJ+Ddv{uG zs@xr!0IG0(FffcWn>`WE-tW_C?w(wNZ=%H-N+>O*ZNdv|7~Ra-NT61HLS=FIQgB6& zBIE*M&6J?-Qb8yNm3RXfk%|3HOkL7ogi69}1u~MAwAqE2)?L}hSzgpoKu;roiAs!y zs9>_VvF8`VSoXNAO|Pv5j0-vPJnTz;2k7$C{F!56fyzEMF~<2@R17?}tND8Q_WHto7WC!% zn#CBfs?^+eJ$rw@I9os4eM#zhPw$FH5%uZrJH$#T9>ynZ^d2HAiyjV6Cs=Sa-b%iS zin<@`k+i3b`b5}_jd7-PCNY~6j8ad;PmBtvcrA*wM{PK>$h15!hk#fKx1u%q(Zu6c z^kA@;N!z*VDw*Q)UY@By#1c zcZF6=UfKaQCx6Wr3%0xSt7ysx2ypKcU;ziGY1PjLI*8XdxaKWfT&dr^2?IwrZ4dX2 zz~M+vgi{k=jLa3Z7PEzWox)>SA9Sm>C#78XsCy^Z4a)ZlDTJ=5>{TYd(ND2Sowy>Z zhn63GHjD|z>TSzl;jANQ$8-W{+AErmxJ?K>Iu`_89v-YCes|0HMX=!bh#p0CA9=%P zDHJ!Qf6rOjZay|1NVg{T88~IhW(fsU-RYwu!TcnwQ~zw#Y315EJF5Znp5MEtC-m1F zLVr|7w20S8RhgqIfK^eQ>yxpuv7oC*OhDw}63g4xC0O0?r4B)bqZKD=0GzITFBw5O zEEc)+7I*Z~;-eL2zsL*v2oVy04*euQeoCm0VX@n=w6C0*zlUGk81#nIMZFD=o<9uo z46=xh6aX&XvR-ZTSR<(F@Vr-%5jb}^;B0oq0Gyc{8>AM_OmE?#F*p?#L7*sDckg!R zEEO=-+Pt$tx?(?zu-B1{0I~(A~Za)Nek5_nEEdbHB3G zuPYQdOy!&QG;a1l5Db9=mC4IfT4#hMHUcuG$a0vLeKZc1vRbIXM8ZRR}^+KuIHud73(u5s1-94x0bwLC-o&KHvKpvxoRX~$mW zXw8aDLTL(+6z5n+mPX?`))+P*bjnfnVbN!11##-UG4-6<>Qf(sMJ17dI#f*LLFaBj zncTpZ&9p4mBVCl`woePfqh#<*XpG3D{w}7hoE%Z?nuxNDcC`UyAxEMpdAhe_4<1~C1b5d!a0~7h++CYSf(5tW?(Q_+@OP4Z z&OUd)cmMalH|}_A3>K3mbJna`^;OlZT7d!0HA#&UkCPUwA=>)*4-Qq>{Ltt-?Tod|LgAW)@V8G&wsU&C*p+M z_Sy6r+(J}-iKIk~E>v01Nf3;+Cl z{}eTqFFpM@6(KMl*R(X>e!YUXh`gE$b@xG-T1TUe0zcs_lxjg=;2 z%hb_|XyE5X`yDNGO*{1LsR7cqQd|X10Uow7k}>WoYM@S>&KKo08O$ZmFAX#=Gg$-% zO_4gE1wISZNOI`wH_uNWbWC@}!mi`%K~M*noJ8EC^|x*gH<*De@kB z+NrdfC6|?CJy3&u6RklnMFSv%?j!smpVyM-q84c8iJs_Lq3Pl5-?YQM4@7v~k%(0! zWCizSird%pYnL4gYv7}W?JGDk_s$~rX4a*+JPS{SBG46NIH70kYA-*vqb;y=)s~wO zEhtl`jV!Cj5+|^P&}R<8?+rwX9i>y_t%Oo1^QU_oq9@hu^lR62=2wF#u>kueZQ(SD zkLy~Ry&gZK8%<6tH%krM5WYl$+fO)GPgFloJ7>$bY4|hKS1&#Fy*9#FMVOv>gj4!> z@N(hpB^cQTSw7S~>=Ve8pN4or=+F3_tETFaG$nN@DLKvTE#?QBoUMS7Kk9r>dz|isj3H8UEL%0rWtwlRw zxtLE&T#mf&8+rx~f6&eH?hu%pnIY|4_PT6bGR_t3O<5*eQV+%cSd=h!EoP0XUQkt6 zFS1+8&F^TGk=^e2eyF5Ak9gzA*2~pN7P0 z^E);>Rk=8#PTW+Vp(WnaGAh1VV3LN|d9_4``*JbqXN_A$GxHI@gI!2vlBj$IRnGT` z@8r>gImWMB3?yIcB9)h0tEtDa`Wxg4_xVXZdM61qp`#$Nr{0z@$tyZx1CV>qISEld zhK>1Jm85Cc9nA86bW!b#fq1qiF&Fa7LP=ut2bxMqRB`gj6HkkwQodX=RXWzu&aH(H zXNm3RbHXQcfvmc~a4eum>r83+VROCWAXmNsOiEzF(yDRqe~y~dLU*%vUw>QKV(E5w z^J32Mba&Q?r}BG_Oj3hsFFK0uQUdNFou`DST9Y^QtZMT(jKKaP#X0~tGkq*BBf3kn zA`B?_SU9JsyG`LOxYQ_My5aF~_Bkxkv|+P720R^NBQ!9{^RQS_qt4uqb$}i4RKv{r z7_^0pr03I|e0x2iu6i7=gq6xJ(O0hLb(@P}+F*1m?YjLFBB_!m(1(?o$$xp>SF%s7 z)(t6HFOYpa4OY)2#BFNvrooO~Pn-_d$DUK)$k zYyvti*2B0LE*|y-_&z@`Dshq?7#NtON}6(&YEEIKvAtJ4GXCIt>J1_GvcZ5Gol#SF z!xESorH#oF>~u1CW;pADudAbzmLk3SBdvn#oYCpSZfA zlroF%6$}^MlL5<$zJAURaz*?znGCwarU3&6o=z?<4y(1;;$u^aUr5SA9h-t*Q0p4t z^HO0bYkunwDhc4OV(Y|b`vlzaO24{>k{$1Lh6);8*e|vj;AXJ{y+G78f78N z6zJ6`K6G_&%bUyX(i4>%KS`PHp?gT~AZkY~MXZtayvSa2D9sRJ zmEwoM8f0~}ie@^vY_*c2>fl%>lrLrnd)2_{@SA?P?=&CQ_jsNlzx62{_*VF4JDa5E zG+t)_v~{}m#9d3>ff*`Z2xou=e7`39wD>EvLER4GYSz1NMf|Ow3e)G*8(>F?KX z*gg~C-T2W=e+p%jQWSL+I4vw<5VkJ1mc_L|!9i3M>vDtGCgssqem=>SblPV)cNxK%iS&HUN%E=c%_mfhWj8Y4r z8CRQ8YAw+6N9I)Ps%2Eo@9O)OR3;*6Ge%WUMxJc|F>~Pmk!sjk+5h8pN9A5TWhy7P z_5L?fP9%qAbY`Yx%Iv*VOZ7T8+VN46u^Wl z+?}B(u0083+Mq?2ITeFHE40iJsgF)5ze%8CBn5!SGWNY*l`=7JIqH01G@C)6QRMsLFYjoi_(0Me)-YnJaeL(v0 zo9@~(xi`ZeHawM-UODvrCy4mb-}ZW*P1TEhU-oCpHq8|V;TQk7CSIqJLnY^q+}dqN<;2GlWPTx z+pYA&d`)Zg8bdP`74U9+phkJZiB}T|d9@IC9R3D3te+OML!@#)z-Zpgj3{c)sd~)9 zvFbp&lxYS@`LSRWKX2__me5PF>Y9GnWe*Hy@4W&zrumUYHcnoOmw!nx40Di%v90^Te0g5%P_kDUC|O^UH@ZY~BdgK7Ho0lrI_4@ZL0>PZ6WuVWFrd z^@ZLZY>{6Gn0}(GT+J1f$>g33Ib5MagBDFpNKahnwwA`ht*B<7!5Bf-j-B0(`!q_ob4>Em+hr7Y^3FHSCfADmDyTE`vnC-{{GTBI4(=I` zR;d`^01Fl(X&0nCRX@C@@NsEN$wE$4LEdh>E+_m0+Ft%_aoe=6zAzSs1cWIW!}MHa z8bK69)dBYdgnDc-Lg=S2MqXnx9r^SRIUFm+ngh~i(1m2`5{kNG_HB1da6bE%Dx-%$CzEh^><6v9@WCI+U|`+sAW*4 zagt>vKb3ZUUxEG<%D)t(Wi)N3dGPvrTFYcVrgm=>alqPxvlfj~)QL#i$*eOR=cq#P`H>}7*UIo$a9tNvZ585FU- zq;0b!GSTGdNhjRJo(*Ir!$27aP;i}mW7yIpbCVOB?K62i$8!6+UFB{GTbk`{{AkVc zO9^vOfol#oG4wzyhm8kBX2~e7@*&+EtQT-&iLh@%p5re= zkMa6Ts0y3J;-DD*EE#*#LDrmf95Hx^v$loQ`vH!57z&28>$OQx$mjTa*S~S`i!F2z zeU&&-sRG#zmoR~5AIEV7nl&Uk`*JC^UG=UaJ zbC{3f$8iJ~(vU;6=n^n19RCAIyGSl{H}>pW-G#Hg_Rh3FN*{0OkhufDc%`x~ zr0t+E`iSL0G4qYbCwQJNS@h=}?t_^{ugvz(w%1%^ONAP8dF=SfdCXRLdJu8a;E@4! z^V&MAK-%Dl)ECQN7`JxNMcR58q&R_#Xf4@KDZ#tu3tx$&Sg43G@-IDyoSz?Hr<5Nr z1TE}2)#^wNg)GmF_3xTp?XkqD&Y!o83J34LF6K1pX5GE6*W-t& z_Be{{6Jh>x)B1>%W}(rgeDeaa`V!A*$m0m}|KjUP+j4wo6L`1@3^vB!gmx94g-3c0 zW8(~jn$EcZHkn1dPH%Y3M)A-G6C+!VlZb_j#vT>>>ajdee^$VoP<1*=-_4YK<1m(S z_PG3ZFISADh#Hs<&sS;Q`&ooow_=_x)rTJxhN|TwaYILhQszuzL&T!I`8)Lb75bnR z@PoXaTkql=Jah#>-fphYKFLiIE>EeO{gP$OQ^5!5W3cNVH{M_SOfef)fExzn`z)oG2&2OzlH+uM2z-WqxI z-#`!4AZJTN3=G8gKWRkhad(9lu^cPK)3o6MO9W(qg-k^_z{v>9e~I4z5wlbM4(|UI zwErEx0{~fnwgF%-|Dz=gf^9VXzV^H2>-+=&_`hob4y9@Hj=m;0FtqxhzFKYz47R;Y zsA){?1#fTJowsbIZXSRimk4f;z81B@S;2N2SgKt}#zH>Yq_$@=FR7Y6DKVs!V~d?D zxFd9X5_|I0+^vUxb}+8;H{u|Xhi@s+cg;6T0mWUbcA zrhvN8rJo*wDtuTu>LS=ZnCwQ(jizejtkG~ACCh69>lJxx{1K0X!TU_)zm9M1G*d{A zSjTb?_<(02QF6%7N`c-V;Ma(zGY64I=FxtNYK(+?K&EKEM<#2S@p@;EaVvc+fZppm z8%(D}79RyTu z{&nm?`w%Vc2upp946yejap~O2UfxaZuFlc`uo~=VoLWANL1g*s<*$zGf+=ct!{c*? z-L`2iDk8(S=?|puXIx4luB3NB{Pdsm@}&s_Pr^NVCp+N#T0Yo#kl{xJ29BC#4!->H zNu|^UiWiV0P+%G`^R=VT-OXE@n~s8zghXw-4|)sFWyGSL2>A%v`Q)&oQ5EZ0ROWt6 z7wBG#dsWI>-Z%52vueVjvcS~PXJJ1Q{@@+Zx0D->HZD#iNnG3a!T7vws?w?Lj z4T^|M+-LPMlc^WSb{9q!bH{qauMQcAM@Mi)R_f3_rxDngE%yyqh~ct1L&dz`xH;XJ z&sgHC3LtbDd=<-hu~I0cwcO}B`8^Qj-qCh_lXvHGVyx56vB4X&f!qg~6g-W85ygg# zbpFo$2=B**>4ei~bw(mSHc4oEYvmytAo<+^Mk9R~Q9o`x$_k$8mPQ>hJTf@GOQ7$& zK%o|3n2++a%UKQSAMtGf`o_mKB#vI4eHhq^Kq;L;eKdXPt9dQjp;@k?NJM=fKuWtB zsK)jJ+`P?Gkbxj9u!mk2>c@A{Z_*h2O3?SZHmNs4@rwytokAIFebR4?O=2{FJnG7Qc7O9?QHlQOnj^UW1DUoBLi^x0=5;E|@tk*w2b z`sJ7@ugDv$L~mn>Q+?Eqh2Jvx8x5?lx>n0?SLz<5)F3pk*Ifq&l;v??MJ}_mk;T$kk^O;``b> zE5G+z<)eN>6vl%4@%(ecw@s`&A5UV$hchJ~SJ~9Tl94wN`vV$REJ!N|9_qjOOa2^* zx~zjiy%4UEw=AW8v6%k~4w4%)fE($1y;o~8=dYGW0B`P~*D+P0WY1Joi`GQ>`1^N?@vyRa-xF-mqJDfP)P;L zUYkm=BldIZ{2)!w-=GqXJ6f+bL(`0JMknNSuCf1F6!b2_T|#Qw@gvPL$c6IHjG`HL zg$Z?M$?W522O>jfb|K}W*!awWJ%V5NhjbHEujNOYpR)%{8L21(f~f(Bwx8rQ1Hx@M zGX?k{U(GItT4qda9;gt@aXi>-N^j-wY7a@3TM6+s%0}N~dVt-*>Gb=6d>iuYN!?5m zID?AC_$SbO*-KkzGn17%1%-5s&EdSNiiMK4bS^DRy_C%v7kPzwLAG?6JK@i2`-Vu(qR6D`9QmKgjNxhK1K)3i;!d>3C>@E_?TNn!3rq7< zdA%(c$=&ssHOEk?qO`9IGn+CrKHMz0<2EZ&07 z*Rq^34~FBEig71GDXWy7T>=aX=FseBoDvVyrA0x=elIL(kZ`^ADf`|ur7r4Pdg&ia z|B-#sV2~c2t8nS6R>)6~LP0Q6^s&}d!uvS%%w2;?^a@88xoc;kguih3-5uzWSP1`O z;`+^>x5W;0{yJ4$zsB7zSe!#z#z|yL9a9SR!KT?Mz0n7HaV8%%}YiH7;)4|KdKDC_H_L4JKaq z?TX$Skr-}5lu+EYO!qX;jZ;{L+a+#I{FbJW(F?a z2tVESfJjqPyv`Hv;UAPL%tJ@-CB zCg4V)(;C8RmKwVxmijfn=69kyZ_{0OD0J4IwBNO zPLg9=yM0lLLGTZIC`|PsY+b)@ICB~#A1s<<_hr;#8i{@(21_Jd<>0R4QEaWZ^&<}q zF*C(-V%_IMChePi?T#CHKai#fT4)OPZ$l$rxi7lg)r)elr4E1K*-LYY6n)%lbZv#- zXiK|&aHZVLsTsJI!@HHQz&*xcv*WIhl>FSK^~|?4AAk`GASHuIe^0=^CJx+SLC&E- zgAt1G{Fm}|tbfy={=)%|5Mo;IR0T6@oY9$bx;m}+Z&D>xM*wzsKSDS6e68j0!$M<( zW;gEkJ;@MOKk4`=0kTqLmX&;nX`!B&h`M*}E&`V*PcH>GJZx z8lSE;Lo3LRGs8(h*o(D>h08BK%e}E(_703dto)A9KT!ZMoK7DW04-S( zbAf$0*~3QmdU4xGFE1}0lK0E)r+8+%3?VNBg6c}!%Y$ls(6e_@Nn3f}Xqw7VBlb5_ z8LgP7sMRD2>rWBo-s^&LE#_ufX8O$QB%a!$tPg(D1ega9{)JQD6eK^@O>avI&sew^ zi!6VE*}u2q2ar98b9vg+0vqyRODUwalYX>GrME~eOJ?9*oC}l{a0sv~){2C7=`z*u zDy@1ZHQ9L5Wv;*qDfw)3^qMhf5jO(fv&8$lWxG2-qAdkCXew9r$FhH{vc&S> z3(I|rWv;tRcgcDIx9G@b{N5jFZF^>oU~%E6Ha(4)w+1L(AqytcXmi$YC3hmAc$??y z@oDXnm&(hY=cqf9D>^h97cMXFk32AVW57VvXCS9jG}xmG@%@+U zdV-0T_lQecyh@@Aw2aEf81vUBv&J#+vRkey4_6M&VYvN=6So^PpmC2?kH9~GKI1U1 z;Q!k@4G#WS3^8W6;ULT%@?w7>#PLcqCeTohvwTq7v9Cq!+*pcTeGc<;@o4)8oSO z%a)w7+y5EA_4bxzHRuHWaWIkep=G}SXk1NpD)Q%^5zF|LBYQLS))H!LMe|wzQz`SI zv0s#B9o}n7s~jT*_=R`|2~Nhm>A~*ju3z8COz+lc0Nsvm&3&d;qUP7N2{6KtE*xMx}J|2(rwl4+niNkCb+H<n9y1Dmq-P*3+RpmWbtcGw~IP*1Fny9xe^# zufk0x3O5i1(S9qbbmc3)K%Hd#JKu1)QXt^8R)xtZgASfVm92u`rsq9Dx;b~HdqL3b z?3opMWy+zD4Y~3sl??94^Lx_dCu(R~eOVU^A&NAKiR-6!in$sYAa8gj`1b>nTE?7_ z&?*P*(n0mc`jq(_LjfyoNY3z$TY+7i$WHD49i>MA0ap9-sF<&fIC`8T+a-4|8kH$? zmitT9*La4u#XsWd%kht?IuS8rUkqBAHn-|3tEw)bNE(>3KjB1!+Fdn%8a^JZl9_Cn z(n7B&HK{VBzHa=zA~Xl-;pD=k@#50`ANn6@$#P(UOu}QVlU|1Wv`;d5oMV1W^ecaJ>tAj+Zng(d z3+XBYx*}w8?zn((6H##*Q<#=bj8`P&*7ZS@cP~B@oaxDXEi?BKf4#0c=JMKkBCzfzUz6g+%h^RBq%B!t zoiifc6rk5fbvm=loDz>fX%A9o?e|D&uPe=`s#^aLwzbM@(J*mve7$!CCA+#Jt2>hr z1c)PBhR7LG9uqe{TX^?I6$c3I-M`_`hpUY69e7l~tjS%3yNfoD5%gpBMpvgW|)BFaaqyOX>|GPvi8PB6oe1zaFbQiCozlsHoPPn3^nmbZclCJ4q1r{j!NKtV=9V-xlw^i?yLJ@iM1?AlWMlXCrbesDU8EjK<>n#g%~%GpA_ozricllKO;M_$oR#5_*JP<4+^3zC zcBaiN)#0>A4=aFq!B?3YhnhJEppG*&W5ZS*i;?2~v}0jugz0IWqZ>RQHlV{W0AEc6 zQ5d;n3G#B@*`4M>50&l?p7t~6qxJgrYHr*pT?OyadB#;K=QE~I_+zR&-E$kQ6G!%Q z5lZ?KJh(2kb8HaXX5w3!m%rovZ&6muECW-kcu`>$@~GWE)$5V6stZ=;R43o3+!pU6 z&}@ts(s(ZV9s@@CCJB%DxOqAnk+K#TXaV087oSzVnmZgHMStnArph`<;tUzKXlO3Z zeDhhfB)*D)M+3*w*=KEur4C|B5U=`%+OoD$0qe>h#33lHL9*m66wh`pP1wCFJ)LxW=IrZl z%kL&1O8PAAE$Wf)!wltpf9tK_t12kLmnpENKBEVbXZ4V!QG)z5e$XgEtTSdqoE-M3 zp6nb?ZZ(oBQIRHX6jI#%R*~V31T7Njb-*+0E^j>es<^Ym;X8Wh(ArvTX-`>EkY z4F{O1&qi397pdAe`B}KAS|?IL=Oi0!}nxxYBx&|gEq#i zMTONJ;NJnK)(~C9Z20+^rZdv`f(~_|7#6n_VYAQkeW(zhtn`d9tQ2pqvjnSJ$M!N&mz{th803+YJ?@bo!b-?IQ@=B`c@exUD zoDk1DXMNh7ZsuzMTbU?YRG$+xD~v#Mle|!SrM01CuA(f_Adj79Z8aspdFSZ>Fq&M> z=Z&Yj@|-IC38)A7Jw;7m0Lvx{MjNcs*)S|355 zn?8cO58Jo#R_#C+^K>z#i?d3B0bST=abkqX;xh$Q4^~d>Dxt=MgK3W`yD=t?lz1;- znDG@qCE6#`5 zv2pZ|r9_a$Z)cxVcXjG}oc8x>b+j7qp8uAN9U^F_(D~-}zBFCzG;cM&b4ZJTTRLfp zej<%d>tN5>?ovlkJ}!t;(G>PMFt^F|L3Zt1Bi!CuS0W&f7~1k-_V@A zWHO^XHS?hLRsU65Mv%%@p~Dhq=nUDH#=8yMWgbiNFrunsx5k9NolWWWjSZH}Zh`*T zbaO5EuZNN7D@W(MKG()kYbH|Aa3gp?+X2!?Of@)Q3c!gk1d9PvSo z9xo-WJW2RQzT%a}g1 zJ~IV8IR%!?68sfB1smq&by#H9m!wWLYC|yzQ85WIG0FrJbW+yKT&&m-JWLT>MPnSI zG0c-?gqLs***PxeOnz9*AsmyABep&Fo(J`c=qDbwaf_GJBQk}0cPDN67k86k%vib!b)CCC4xeaNiP~;C4!B(LX-sp?sXg2-yL9C0#Egqurs#{7_RktqOs`I zHEqbm#^T-)X}zJkGnLDRnBR3}0_49AD=LiFI+;zhS(CgG67s&k)N7Brb?1du9&&d6 zUy}{LYYP8PHvHZC-=rO2m65;i0uF!GNM5ZCugoW+ujTiw+r3Ezmz`>Pz;Pk{qV&Jy zEq*`zAFutceEi+PKU)8vsv-Y3iIsn(Q~ui-`g+)zjy@(nvUq}$}Wyyt4v)V9-ipWRR}>8j>Cjd0RU}w6xee7 z9&sAwkt0tFGH%W#`2J?AQZ=;GeRtV;-OsZAefdt&Aj<^`Y$>jO)LOAiJ6iTH+zP3<(UPzAQ_%aMeE?y##U!9JiJP?-jMe4)=aMy{D} zk((_gciPh16h`Z(X<%er+w4(XjH@2LWIx1oNRw5bBek<`PupYk0Cj)T)BZl|%*2A% zd-EY+a|)+7x7++<1HNfs#Q@);UDw=}A(f%y;ufqY@1^)6#gra5saCZLu`b)Vo(eVh zYeyJ`!9l4N7J0I1uxD0uPcEM?0-54vVz{Y^jyUlbgMVxfPf_=JrS>Fmd{^NO`H^d; z4e>3KDtNGQA+MIv6R3FgaW7@)82eWOj?I-{Iou(`%i+C{;EgPKbHtn{{Yx9Ur~Y6@GU?GEg4n zYKoW(33KzY4g$ngp0@Gs{k7il_66rHc!}>Q`YqDst=vs{c|RmWJ*f?k@5Y*((*-v) zY81mua574{Ud1+Q^DgM#4_$JE=rx$I*^TY~Vx7waC?zR(+DBbECe^L=O7)GQDP-9h zZDPN71jY`s!-_%1C<Wd#}cvmr})cEH^VzkJ5g(s9^rn zRLJcS(v`YyDKNsCaMud#Ip=W`c7Bzg+4_mIe4al91nWx}L?tC=#IBy1xE0e#qrN<6 zImCE1rGAC$;=szel*tdsKPc<5T#BS}W5q_5M?l-Dr{&b>5xXBNx%T zZu!PTYrTLEuPvTvsm1MlrNbX|vb!kw>&Z;78!GR(M+dF-g9035h*Mtr9yK7%PuYV)cKZgG2 zfdB7@{x6m1{}Wi-c8=?Fife~-GHaR%z0!MpTrYS5@cj=2OlnRivgVC+_*DV_w~hL{ z^?!=$e<#j;wf@e+pWwP6pcC_7v##fi?bP=uxUhI@@p;}IHV6u=KdhZjXSBHr!hWow znHoZU4aXG%Fy14zc`#ek5hyosdPC9l?#=$oIs=62b)*{hyGwrZ21Q@)+t_ITJeEyF zpWwFfvGV4fGFI;aQEHKH6HB!xp?3b*ItxF?3nGvl&!l5 zTj+VVqIKjnJT$^#G-*aenl8;E{g`TM3xN{#A;h~=K*-N&QUH>ZS#lX{LPsP zYs0Xf8gbi{l$=K&hGv~TNpY#socSVi+Lg(6n3_$H*mB$#>jPR*_?p<`Fz;5>6$YnYX7Z=&zM znKP`z+xASct^=77rV?7Vg;IcfYuN4b?+k5bw11H3aeB59ozy!% zjw}zw>IGvx8X7-@*tvivl9>|NPSs)B4 zIq0qObYqGfN|do9_jxp`QVNfPJ)fl`RgD=f6VphVT;7colO*kto*w)?@w~f{Mz1KI zqUW&88(%W!9vWnqtHflyrrpA19EBT}+(-)@IoIrQzcu+_PN%~f71o?eE%q|Mc~bv!ur46${Hfr?ZbYBc~iaC5R%QuF5_0O z8&cS{qyGOt9NUN}m^-_&@aub>{}w<00oZ?o`2RG1{sZ9ud)soEoc03FQKQPS}5X_`GMo%DQx+tQDCy|>numDY<4^SSZk(j zd#3;c$}F9UGWSzI`zAHLc|p7$Bnc0TZi>2L;P6F5#;XdY{aicqMl~j4Ga}iwpxisH z38Vy*eCfP3PcJKu819lp#31T zdp4@O`T0%`YpO#V67VAKT`Y__qvbH?id;)DnkdR+@$zij0aE6z%`)pX8%Y6{Xvpjd z+2FDd5WRVqrycxp_WRf&MZg(#GIunw^ZYt@_0nFxU0$ae^wivga5hM0WGbVKpP?&j z-kV>&ZPYQxmHuX9heuwOSgm^f^nxeQ3?Fm%=E|w~{WUPQ%Q$e}V#vN@P(G(c-fQRTwk&lp$J#^q#>g`M&#U3K>tO*NAe%+<4I zB;`(M#SydI33{Inn9_(t6VyC5r<@Dox;!p5QzwdtB9Ztk?jrWInV8sjo!nu*rPF9~ zUstpyR0^#6=n1~_Y!Cu{2!9b{UZWhr-Z^c-x9BWp>`;XrO|`u|4Y?&4;5*$ZF&C7l zl)A@?(H1j`X)H1_aAJ2(;Hpo|R6mEU=E?%(tyx;%$ds-Pe>h%41NwT`sx==ue)_QH z%}(TY&~AKc?&;@?ET}mlIzbuk2zd{4!MrY=Kn4_;7^2J`rl(dts}Or_VCG|SWO^7p z;|cv17osU0oUTwy0Q@V*&NXZ#P?0|SzNs~SJ z+`UaGEq9`BR;^+hW^<8b(e_@HU{+mB>eeN7B|Bn8>tZLq0}ZKTM)l*G^bSozyp7Gw zitIfX?VWtEnnb->cFOC}*3uPC%LMScgl;^VEC~lFo4DDu>4z8_K(qZh@p+PC9*~-V zxq%u5ykx7*oz7xanWqB||ab`y-3w+ni+;Dw{; ztu`XLK|fKts4<_>C7E*YreLD3Nn5O{RVmnYa<0vhSHtskD95s^asGOCJ)kLBsY+YV z?C^^E^sNN*E1_d??1WZphK==>j$6yB56y*6*B#cowk{ZoNqxOfHsdQi$|Q2e=I+dC z4J%ukGHHjH#lz>dph?nq;7ixrs{8W*>huU{u^aIMh!L64!qO*6{yEP@3(icA6tI5< zJ?6dwZ%%OE^%ePZy<=y5g$({c?G6F+wy6m^fwW2&pl}aK|`n?PId~ z2*t`%nZ@X(Z5ca&GJAUQoTm$nxih179gn;z$zk3KGs$pV3i#n+zk(~1QMJDp=gu5@ z`h74As;F&XQ#M5?7B^jwUvt{}Jtyx4v4Hk1N36DcnSHkN!}pEd?-O&oBNu`i{XLC$ zP~cfzope{p(J9$p#}Uj>4<#-g+1&;m*E^Gwzs&K67WsSM60myFcXGsV!y-lYwo5~c zM_Z-&qbnDJ8&~Q+oadO!5mdj_0hCgnC?B|8{$p_=gknE)q^vi&Ck(9VKvwQh3BEZ( zu649-Zm8nY*8b5ezWa$=+Wtn{uHOB^18i4Y*;aSiST&)&R1Vooc3rmh*yNh`4-Q(c zHBY`4UhQ-L$QrT_lzL-`-FK50EaEO$6o#1t451ljHYQkv42yW>&zPZot9+9hbtM}~ zJwFgNZ?{B49$FVs=kLBgnYE~G#kKw;C*pBZJB=q|df~!u8YV&~I&#X;#YF#L>AEyA z6i~W|n48&po2Qd`86jrH++A#=eA{H2yp5-m$;Tp3Piqeh+`ORl*`_8_P$zf%hKa=p zSHBH9BlDei{-!$S>=4+5w&`pjvpg|>T}CjT3VVq4I7+J0R;u!Eqg*IYx1PUS~Q{ekdp&93CyNO*zSbyu{l1m<)j)2 zb2iVM)vy{||5DMlxrMvIwz$4pDR>8baJh~ocj;2lDWiakJIqin$S8)A0&`ckDU;3W z4jH&EN1V)gVW@(uShx|}sBeF=aebGp*1A+Ce+<$-8?*7Lywh5((OJcO?JZ!Jskup-aK6nnp%nmmB-! z<|ihXY2F-j3YX1_=vZgHHxz49R#d|>CJ7`xXT$QQZDUCT-9`8JJ{tWGCZIdlauwIc zTJBg9qJw%Dcg(69t{3a%cI+lk47lCUs^n*FHx zHXc6B+0s|vrgg2OC_op+tWx9dJotPs>K3+?Hm7|~y=un7XcEJc&wD|W_OM79P`i)R z1GeT`>NgocFOgBmV2I0CXAHbT@8}?=RW(c2S8ChEeNB)iiL3qb9z}W><_)DM^(SOnl}?V3*lex6i;-Gto$DAL zCS9tz?zrP+mYSfB*8V`tEEsO>(4gmYr5nSoRE|AYGH}SPv~<5E&Pj3%%wtY`zIeH! z;q^uyo4!u;;wd+0Pi?QQ38ru%Mg8MjCLeECl&lB=<@JusP0g#&pa3u$HU*+Rts2&0pG`UT?{r+f!#|h|`2VY<99k9vpAq z_8l00bg5#6O0PA9q)0J&)pd7IT}(|zlH63C#k-R^zu`p+iVmp^v(Ss#0lV17QbcLO5@_JuhC zq57g;&fs8=^y<0Ey1ttW)j0z5$Lhysz8x!0cST(gkS(ec5_-CS#66=)B6%ZThMfqD zB4H{&_)|~C@Z@Gr_ukj>=k=_8&SJy|O)ZA*iqRd&_E1OXP+c+*HbxrAJ|Oqn@DZ{K zyjktFJOmTVlk3r$tf|w{dY!vhs0{W-c%2sfrw{02N(hsDH&Dz*D0|bjL9%BRw$Br` z8zn05@R3%}mqPuIO+%+ekLL}a?qQqCFk)${VnmtC>9qUhNCwZ`@p zZ8NEhfhigKJKayE+Xc3|N;sKbJ|)F^P0MaqtzUV*SmwNqrT9|1x68+5F1A&r!^)#X z(Hm=K!A)yo7>ZAnUtH`wm{QMQKbtKM1!lB`0a` zCr6rsVUh1!>A)^@)W8 z&3o);G1t&@$Ua->Z-3iJ2Ii_sSMO9)bmgTHN63nn=}3j=@@A_^tOR~&LW`k!CLk57 z6F5NY`B9_m?Q!{m!3E;>&vOz3WU>|z`M?R9lLlzLjZyhjqS?9$6>-+zs3eS}W)a^Z zQIzYQPg|UFzQDwpYseAh#-&Wmm#YCv)I?0XXRP>Gl}`hSDUEaMq%EH7C?#V)Vt}ze znK}HV5YP>LVu=i$-1zA7X&vhUvlqu`TM4v*EdF3R+qu&>LQ80Xyt;@Z@aU2x21(nO z={8Q=iEwcdP$Sv~v58Nc0#)0dUREWwiOkgtNqf!oIS4 z0a)pYfG3Sa&aUuu+g~2Ox0^Q=WbKrrx=#C!8HmxacXJdfeM10vA82rqePV-zk4;Ci zP~F-9Z)1LQ#@Dqsu_*I)#m&JLviIuQiiFMGEwfXC#wP7{&COot7c_zUhILsYxHTB} zcbLkLGcJ-qF>^!9y{IeV#V*Qn-It9I<_1{pB$?ccpDOXWiXz;out zjYP>2+H0cXJfH?sh)$%8F!`_xHz{STQoZz)UqyZ!tY!1%Hh_=~Ko^OC{XNX0q(MAT z?a^*NsLutNssu z{~B3B#peC$=+Al}ixAtSpXLebB_kO&I*Kx7?XAtCo)0?)EN$T8b4_5+m_4HR zApc_a$?HwbM8;Ua5SaruwW z#ROc^EA*)G0H9d{pfEeGMoZ1-?gHbBU=?JPq^~>P_WA5ELfbJtca#co%mSD%ChqtZ zK=j-cj)k0y7<@DT|UuECe{lY0_0C+G#bmvB7qRoYY6iRv5j9R3$ecVqzWnPkXv_Pj^)+F=Mh z3po>vEci)b&*PvLO`-$(8{E`3)z}+3GJP?^x0|gpT!373W(>g_v)WR~$l@VX|$nr~}ICt5ZL0{VLVHITwbV#AFUM zp(7^S_{1Px6k5E=a4U_YYW|HPR*F*P41>=^L7A9c@$PfqM_i?5tkbVwFzueh(5UdK zi6)gXP^v5$ABM(lUz=nyHYgL5IK}B@ul9AsCZRD?I(^lCI6-dE9RgH3)7muJUkfeBd!+dU${8|Zvu9@C=D3?DW zo3=9~rbWLwJlHgN@~bc>4ZZ+H2q8#+t~A)<$8dK`r$hr4&?Nt(O4)`^QAe+3@K{a+ zHdFrjJdx2RZwsylC&MMxOG|k>mOpD$uUcAt(5L*RRP$}ABz;sS3%SZ8>N6FD)Cc=; zWKKZIPF04=TgxzDeHgw)tlOExbIF;<`3eMBOa%alJWTm(zxw>!(Ed8Dl*2CmeQNym lA9jA9j(B0u`~E*ahu8_kx!hVW?f%ufl(@WDxv0M1{{nCZ-nIY$ diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index f1a4351..a379e4a 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -62,9 +62,18 @@ (bench1 fressian-freeze fressian-thaw)) (comment + (set! *unchecked-math* false) ;; (bench {:reader? true :lzma2? true :fressian? true :laps 3}) ;; (bench {:laps 4}) - ;; (bench {:laps 1}) + ;; (bench {:laps 1 :lzma2? true}) + + ;;; 2015 Sep 29, various micro optimizations (incl. &arg elimination) + {:reader {:round 63547, :freeze 19374, :thaw 44173, :size 27717}} + {:lzma2 {:round 51724, :freeze 33502, :thaw 18222, :size 11248}} + {:fressian {:round 8813, :freeze 6460, :thaw 2353, :size 16985}} + {:encrypted {:round 6005, :freeze 3768, :thaw 2237, :size 16164}} + {:default {:round 5417, :freeze 3354, :thaw 2063, :size 16145}} + {:fast {:round 4659, :freeze 2712, :thaw 1947, :size 17026}} ;;; 2015 Sep 15 - v2.10.0-alpha6, Clojure 1.7.0 {:reader {:round 94901, :freeze 25781, :thaw 69120, :size 27686}} From 1506747e42c2b2157a28f333fbd70e19b0138f87 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 30 Sep 2015 11:53:17 +0700 Subject: [PATCH 36/38] Tune buffer size, freeze compressor selector --- src/taoensso/nippy.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index ae64828..b1d2ada 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -203,7 +203,7 @@ (if (counted? ~'x) (do (.writeInt ~'out (count ~'x)) (encore/run!* (fn [i#] (freeze-to-out ~'out i#)) ~'x)) - (let [bas# (ByteArrayOutputStream.) + (let [bas# (ByteArrayOutputStream. 64) sout# (DataOutputStream. bas#) cnt# (reduce (fn [^long cnt# i#] (freeze-to-out sout# i#) @@ -377,9 +377,9 @@ [^bytes ba] (let [ba-len (alength ba)] (cond - ;; (> ba-len 1024) lzma2-compressor - ;; (> ba-len 512) lz4hc-compressor - (> ba-len 128) lz4-compressor + ;; (> ba-len 4098) lzma2-compressor + ;; (> ba-len 2048) lz4hc-compressor + (> ba-len 1024) lz4-compressor :else nil))) (encore/defonce* ^:dynamic *default-freeze-compressor-selector* @@ -403,7 +403,7 @@ compressor (if legacy-mode? snappy-compressor compressor) encryptor (when password (if-not legacy-mode? encryptor nil)) skip-header? (or skip-header? legacy-mode?) - baos (ByteArrayOutputStream.) + baos (ByteArrayOutputStream. 64) dos (DataOutputStream. baos)] (freeze-to-out! dos x) (let [ba (.toByteArray baos) From 4765a32e4efe7500176c07f59b4c4623e12bdec9 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 30 Sep 2015 11:57:00 +0700 Subject: [PATCH 37/38] Optimize compact long freezer --- src/taoensso/nippy.clj | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index b1d2ada..b239400 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -282,16 +282,23 @@ (freeze-to-out* [x ^DataOutput out] (let [^long x x] (cond - (and (<= Byte/MIN_VALUE x) (<= x Byte/MAX_VALUE)) - (do (write-id out id-byte-as-long) (.writeByte out x)) + (and (<= x #_Byte/MAX_VALUE 127) + (<= #_Byte/MIN_VALUE -128 x)) + (do (write-id out id-byte-as-long) + (.writeByte out x)) - (and (<= Short/MIN_VALUE x) (<= x Short/MAX_VALUE)) - (do (write-id out id-short-as-long) (.writeShort out x)) + (and (<= x #_Short/MAX_VALUE 32767) + (<= #_Short/MIN_VALUE -32768 x)) + (do (write-id out id-short-as-long) + (.writeShort out x)) - (and (<= Integer/MIN_VALUE x) (<= x Integer/MAX_VALUE)) - (do (write-id out id-int-as-long) (.writeInt out x)) + (and (<= x #_Integer/MAX_VALUE 2147483647) + (<= #_Integer/MIN_VALUE -2147483648 x)) + (do (write-id out id-int-as-long) + (.writeInt out x)) - :else (do (write-id out id-long) (.writeLong out x)))))) + :else (do (write-id out id-long) + (.writeLong out x)))))) ;; From 280019a4bc7279b7445722dae6e5d15993dfba22 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 30 Sep 2015 12:11:32 +0700 Subject: [PATCH 38/38] v2.10.0 --- CHANGELOG.md | 8 ++++---- README.md | 7 +++---- project.clj | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 389e8a2..0b7f385 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,17 @@ > This project uses [Break Versioning](https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md) as of **Aug 16, 2014**. -## v2.10.0-RC1 / 2015 Sep 26 +## v2.10.0 / 2015 Sep 30 -> This is a major **non-breaking** feature/performance release +> This is a major feature/performance release that **drops support for Clojure 1.4** but is otherwise non-breaking -* **DEPRECATION NOTICE**: this will be the last major release with support for Clojure 1.4 +* **BREAKING**: drop support for Clojure 1.4 (**now requires Clojure 1.5+**) * **Performance**: various small performance improvements * **New**: dynamic `*default-freeze-compressor-selector*`, `set-default-freeze-compressor-selector!` util * **New**: dynamic `*custom-readers*`, `swap-custom-readers!` util * **New**: edn writes now override dynamic `*print-level*`, `*print-length*` for safety ```clojure -[com.taoensso/nippy "2.10.0-RC1"] +[com.taoensso/nippy "2.10.0"] ``` diff --git a/README.md b/README.md index 3b64c2f..2223c63 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ **[API docs][]** | **[CHANGELOG][]** | [other Clojure libs][] | [Twitter][] | [contact/contrib](#contact--contributing) | current [Break Version][]: ```clojure -[com.taoensso/nippy "2.9.1"] ; Stable -[com.taoensso/nippy "2.10.0-RC1"] ; Dev, see CHANGELOG for details +[com.taoensso/nippy "2.10.0"] ; Stable, see CHANGELOG for details ``` # Nippy, a Clojure serialization library @@ -30,7 +29,7 @@ Nippy is an attempt to provide a reliable, high-performance **drop-in alternativ Add the necessary dependency to your [Leiningen][] `project.clj` and `require` the library in your ns: ```clojure -[com.taoensso/nippy "2.9.1"] ; project.clj +[com.taoensso/nippy "2.10.0"] ; project.clj (ns my-app (:require [taoensso.nippy :as nippy])) ; ns ``` @@ -154,7 +153,7 @@ Otherwise reach me (Peter Taoussanis) at [taoensso.com][] or on [Twitter][]. Che ## License -Copyright © 2012-2014 Peter Taoussanis. Distributed under the [Eclipse Public License][], the same as Clojure. +Copyright © 2012-2015 Peter Taoussanis. Distributed under the [Eclipse Public License][], the same as Clojure. [API docs]: http://ptaoussanis.github.io/nippy/ diff --git a/project.clj b/project.clj index b4aa224..8443df2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.10.0-RC1" +(defproject com.taoensso/nippy "2.10.0" :author "Peter Taoussanis " :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy"