From ab3209f2dcf8d829b1b985a6a1e1ff68fda516bb Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 12 Jun 2013 23:36:30 +0700 Subject: [PATCH 01/14] Housekeeping --- project.clj | 2 +- src/taoensso/nippy.clj | 4 +-- src/taoensso/nippy/benchmarks.clj | 7 +++-- src/taoensso/nippy/utils.clj | 44 ++++++++++--------------------- 4 files changed, 20 insertions(+), 37 deletions(-) diff --git a/project.clj b/project.clj index ae44847..ef4fc72 100644 --- a/project.clj +++ b/project.clj @@ -13,4 +13,4 @@ :aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "test"]} :plugins [[codox "0.6.4"]] :min-lein-version "2.0.0" - :warn-on-reflection true) + :warn-on-reflection true) \ No newline at end of file diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 2ab1b57..e005e1c 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -174,7 +174,7 @@ stream (DataOutputStream. ba)] (freeze-to-stream! stream x print-dup?) (let [ba (.toByteArray ba) - ba (if compress? (utils/compress-bytes ba) ba) + ba (if compress? (utils/compress-snappy ba) ba) ba (if password (crypto/encrypt-aes128 password ba) ba)] ba))) @@ -260,7 +260,7 @@ }}] (try (-> (let [ba (if password (crypto/decrypt-aes128 password ba) ba) - ba (if compressed? (utils/uncompress-bytes ba) ba)] + ba (if compressed? (utils/uncompress-snappy ba) ba)] ba) (ByteArrayInputStream.) (DataInputStream.) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 902c5c1..3765b94 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,8 +1,7 @@ (ns taoensso.nippy.benchmarks {:author "Peter Taoussanis"} (:use [taoensso.nippy :as nippy :only (freeze-to-bytes thaw-from-bytes)]) - (:require [taoensso.nippy.utils :as utils] - [taoensso.nippy.crypto :as crypto])) + (:require [taoensso.nippy.utils :as utils])) ;; Remove stuff from stress-data that breaks reader (def data (dissoc nippy/stress-data :queue :queue-empty :bytes)) @@ -19,8 +18,8 @@ (def roundtrip-fast (comp #(nippy/thaw-from-bytes % :compressed? false) #(nippy/freeze-to-bytes % :compress? false))) -(defn autobench [] (bench (roundtrip-defaults data) - (roundtrip-encrypted data))) +(defn autobench [] {:defaults (bench (roundtrip-defaults data)) + :encrypted (bench (roundtrip-encrypted data))}) (comment diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index a02f2e5..c3eeeae 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -1,7 +1,6 @@ (ns taoensso.nippy.utils {:author "Peter Taoussanis"} - (:require [clojure.string :as str]) - (:import org.iq80.snappy.Snappy)) + (:require [clojure.string :as str])) (defmacro case-eval "Like `case` but evaluates test constants for their compile-time value." @@ -14,25 +13,12 @@ clauses) ~(when default default)))) -(defn pairs - "Like (partition 2 coll) but faster and returns lazy seq of vector pairs." - [coll] - (lazy-seq - (when-let [s (seq coll)] - (let [n (next s)] - (cons [(first s) (first n)] (pairs (next n))))))) - -(defmacro time-ns - "Returns number of nanoseconds it takes to execute body." - [& body] - `(let [t0# (System/nanoTime)] - ~@body - (- (System/nanoTime) t0#))) +(defmacro time-ns "Returns number of nanoseconds it takes to execute body." + [& body] `(let [t0# (System/nanoTime)] ~@body (- (System/nanoTime) t0#))) (defmacro bench "Repeatedly executes form and returns time taken to complete execution." - [num-laps form & {:keys [warmup-laps num-threads as-ms?] - :or {as-ms? true}}] + [num-laps form & {:keys [warmup-laps num-threads as-ns?]}] `(try (when ~warmup-laps (dotimes [_# ~warmup-laps] ~form)) (let [nanosecs# (if-not ~num-threads @@ -44,23 +30,17 @@ doall (map deref) dorun))))] - (if ~as-ms? (Math/round (/ nanosecs# 1000000.0)) nanosecs#)) + (if ~as-ns? nanosecs# (Math/round (/ nanosecs# 1000000.0)))) (catch Exception e# (str "DNF: " (.getMessage e#))))) -(defn version-compare - "Comparator for version strings like x.y.z, etc." - [x y] - (let [vals (fn [s] (vec (map #(Integer/parseInt %) (str/split s #"\."))))] - (compare (vals x) (vals y)))) +(defn version-compare "Comparator for version strings like x.y.z, etc." + [x y] (let [vals (fn [s] (vec (map #(Integer/parseInt %) (str/split s #"\."))))] + (compare (vals x) (vals y)))) -(defn version-sufficient? - [version-str min-version-str] +(defn version-sufficient? [version-str min-version-str] (try (>= (version-compare version-str min-version-str) 0) (catch Exception _ false))) -(defn compress-bytes [^bytes ba] (Snappy/compress ba)) -(defn uncompress-bytes [^bytes ba] (Snappy/uncompress ba 0 (alength ba))) - (defn memoized "Like `memoize` but takes an explicit cache atom (possibly nil) and immediately applies memoized f to given arguments." @@ -76,6 +56,10 @@ (comment (memoized nil +) (memoized nil + 5 12)) +(defn compress-snappy ^bytes [^bytes ba] (org.iq80.snappy.Snappy/compress ba)) +(defn uncompress-snappy ^bytes [^bytes ba] (org.iq80.snappy.Snappy/uncompress ba + 0 (alength ba))) + (defn ba-concat ^bytes [^bytes ba1 ^bytes ba2] (let [s1 (alength ba1) s2 (alength ba2) @@ -90,4 +74,4 @@ (comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar"))) (let [[x y] (ba-split (.getBytes "foobar") 3)] - [(String. x) (String. y)])) \ No newline at end of file + [(String. x) (String. y)])) From 6fe433b57993a50eb365f0a2efec40c846084ae8 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 00:00:29 +0700 Subject: [PATCH 02/14] Update tests, migrate to Expectations lib --- project.clj | 11 ++++++++--- src/taoensso/nippy/benchmarks.clj | 12 +++++++++--- test/taoensso/nippy/tests/main.clj | 26 ++++++++++++++++++++++++++ test/test_nippy/main.clj | 30 ------------------------------ 4 files changed, 43 insertions(+), 36 deletions(-) create mode 100644 test/taoensso/nippy/tests/main.clj delete mode 100644 test/test_nippy/main.clj diff --git a/project.clj b/project.clj index ef4fc72..0d236b5 100644 --- a/project.clj +++ b/project.clj @@ -4,13 +4,18 @@ :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :dependencies [[org.clojure/clojure "1.3.0"] + [expectations "1.4.43"] [org.iq80.snappy/snappy "0.3"]] :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} :dev {:dependencies []} :test {:dependencies [[org.xerial.snappy/snappy-java "1.0.5-M3"]]}} - :aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "test"]} - :plugins [[codox "0.6.4"]] + :aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "expectations"] + "test-auto" ["with-profile" "test" "autoexpect"] + "start-dev" ["with-profile" "dev,test" "repl" ":headless"]} + :plugins [[lein-expectations "0.0.7"] + [lein-autoexpect "0.2.5"] + [codox "0.6.4"]] :min-lein-version "2.0.0" - :warn-on-reflection true) \ No newline at end of file + :warn-on-reflection true) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 3765b94..64c5012 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -6,7 +6,7 @@ ;; Remove stuff from stress-data that breaks reader (def data (dissoc nippy/stress-data :queue :queue-empty :bytes)) -(defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 1000)) +(defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 2000)) (defn reader-freeze [x] (binding [*print-dup* false] (pr-str x))) (defn reader-thaw [x] (binding [*read-eval* false] (read-string x))) @@ -18,8 +18,14 @@ (def roundtrip-fast (comp #(nippy/thaw-from-bytes % :compressed? false) #(nippy/freeze-to-bytes % :compress? false))) -(defn autobench [] {:defaults (bench (roundtrip-defaults data)) - :encrypted (bench (roundtrip-encrypted data))}) +(defn autobench [] + (println "Benchmarking roundtrips") + (println "-----------------------") + (let [results {:defaults (bench (roundtrip-defaults data)) + :encrypted (bench (roundtrip-encrypted data)) + :fast (bench (roundtrip-fast data))}] + (println results) + results)) (comment diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj new file mode 100644 index 0000000..aa1ff62 --- /dev/null +++ b/test/taoensso/nippy/tests/main.clj @@ -0,0 +1,26 @@ +(ns taoensso.nippy.tests.main + (:use [expectations :as test]) + (:require [taoensso.nippy :as nippy] + [taoensso.nippy.benchmarks :as benchmarks])) + +;; Remove stuff from stress-data that breaks roundtrip equality +(def test-data (dissoc nippy/stress-data :bytes)) + +(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) +(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"]) + #(nippy/freeze-to-bytes % :password [:cached "secret"]))) + +(expect test-data (roundtrip-defaults test-data)) +(expect test-data (roundtrip-encrypted test-data)) +(expect ; Snappy lib compatibility + (let [thaw #(nippy/thaw-from-bytes % :compressed? false) + ^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false) + ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) + ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] + (= (thaw raw-ba) + (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) + (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) + (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) + (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) + +(expect (benchmarks/autobench)) \ No newline at end of file diff --git a/test/test_nippy/main.clj b/test/test_nippy/main.clj deleted file mode 100644 index 30a11d1..0000000 --- a/test/test_nippy/main.clj +++ /dev/null @@ -1,30 +0,0 @@ -(ns test-nippy.main - (:use [clojure.test]) - (:require [taoensso.nippy :as nippy] - [taoensso.nippy.benchmarks :as benchmarks])) - -;; Remove stuff from stress-data that breaks roundtrip equality -(def test-data (dissoc nippy/stress-data :bytes)) - -(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"]) - #(nippy/freeze-to-bytes % :password [:cached "secret"]))) -(deftest test-roundtrip-defaults (is (= test-data (roundtrip-defaults test-data)))) -(deftest test-roundtrip-encrypted (is (= test-data (roundtrip-encrypted test-data)))) - -(println "Benchmarking roundtrips (x3)") -(println "----------------------------") -(println (benchmarks/autobench)) -(println (benchmarks/autobench)) -(println (benchmarks/autobench)) - -(deftest test-snappy-library-compatibility - (let [thaw #(nippy/thaw-from-bytes % :compressed? false) - ^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false) - ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) - ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] - (is (= (thaw raw-ba) - (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) - (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) - (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) - (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))) \ No newline at end of file From da077c6a540ac199f8b2963d560a363f04638c5d Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 00:15:16 +0700 Subject: [PATCH 03/14] Drop Clojure 1.3 support --- README.md | 7 ++----- project.clj | 7 +++---- src/taoensso/nippy.clj | 13 +++++-------- src/taoensso/nippy/benchmarks.clj | 4 ++-- test/taoensso/nippy/tests/main.clj | 4 ++-- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 76e068a..8f2da43 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,8 @@ nippy/stress-data :bigdec (bigdec 3.1415926535897932384626433832795) :ratio 22/7 - - ;; Clojure 1.4+ - ;; :tagged-uuid (java.util.UUID/randomUUID) - ;; :tagged-date (java.util.Date.) - } + :tagged-uuid (java.util.UUID/randomUUID) + :tagged-date (java.util.Date.)} ``` Serialize it: diff --git a/project.clj b/project.clj index 0d236b5..7451f11 100644 --- a/project.clj +++ b/project.clj @@ -3,15 +3,14 @@ :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :dependencies [[org.clojure/clojure "1.3.0"] + :dependencies [[org.clojure/clojure "1.4.0"] [expectations "1.4.43"] [org.iq80.snappy/snappy "0.3"]] - :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} - :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} + :profiles {:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} :dev {:dependencies []} :test {:dependencies [[org.xerial.snappy/snappy-java "1.0.5-M3"]]}} - :aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "expectations"] + :aliases {"test-all" ["with-profile" "test,1.4:test,1.5" "expectations"] "test-auto" ["with-profile" "test" "autoexpect"] "start-dev" ["with-profile" "dev,test" "repl" ":headless"]} :plugins [[lein-expectations "0.0.7"] diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index e005e1c..931b0ff 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -278,11 +278,8 @@ (-> (freeze-to-bytes "my data" :compress? true) (thaw-from-bytes :compressed? false))) -(def stress-data - "Reference data used for tests & benchmarks." - (let [support-tagged-literals? - (utils/version-sufficient? (clojure-version) "1.4.0")] - +(def stress-data "Reference data used for tests & benchmarks." + (let [] {:bytes (byte-array [(byte 1) (byte 2) (byte 3)]) :nil nil :boolean true @@ -323,6 +320,6 @@ :ratio 22/7 - ;; Clojure 1.4+ - :tagged-uuid (when support-tagged-literals? (java.util.UUID/randomUUID)) - :tagged-date (when support-tagged-literals? (java.util.Date.))})) \ No newline at end of file + ;; Clojure 1.4+ tagged literals + :tagged-uuid (java.util.UUID/randomUUID) + :tagged-date (java.util.Date.)})) \ No newline at end of file diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 64c5012..d004c36 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,7 +1,7 @@ (ns taoensso.nippy.benchmarks {:author "Peter Taoussanis"} - (:use [taoensso.nippy :as nippy :only (freeze-to-bytes thaw-from-bytes)]) - (:require [taoensso.nippy.utils :as utils])) + (:require [taoensso.nippy :as nippy :refer (freeze-to-bytes thaw-from-bytes)] + [taoensso.nippy.utils :as utils])) ;; Remove stuff from stress-data that breaks reader (def data (dissoc nippy/stress-data :queue :queue-empty :bytes)) diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index aa1ff62..47625ca 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -1,6 +1,6 @@ (ns taoensso.nippy.tests.main - (:use [expectations :as test]) - (:require [taoensso.nippy :as nippy] + (:require [expectations :as test :refer :all] + [taoensso.nippy :as nippy] [taoensso.nippy.benchmarks :as benchmarks])) ;; Remove stuff from stress-data that breaks roundtrip equality From ac380eb621f83ef7ed888e878bc43dfa2c48983a Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 00:23:04 +0700 Subject: [PATCH 04/14] Mark `freeze-to-stream!` and `thaw-from-stream!` as private --- src/taoensso/nippy.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 931b0ff..4b31cb3 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -157,7 +157,7 @@ (freeze-to-stream!* s m))) (freeze x s)) -(defn freeze-to-stream! +(defn- freeze-to-stream! "Serializes x to given output stream." ([data-output-stream x] ; For <= 1.0.1 compatibility (freeze-to-stream! data-output-stream x true)) @@ -241,7 +241,7 @@ (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) -(defn thaw-from-stream! +(defn- thaw-from-stream! "Deserializes an object from given input stream." [data-input-stream read-eval?] (binding [*read-eval* read-eval?] From 7705c42142036841943ac78543c004a2b76d109e Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 01:14:46 +0700 Subject: [PATCH 05/14] Housekeeping prep for new API --- src/taoensso/nippy.clj | 242 +++++++++++++++++++++-------------------- 1 file changed, 126 insertions(+), 116 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 4b31cb3..5191513 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -10,7 +10,9 @@ PersistentTreeSet IPersistentList IPersistentVector IPersistentMap IPersistentSet IPersistentCollection])) -;;;; Define type IDs +;;;; Header IDs ; TODO + +;;;; Data type IDs ;; 1 (def ^:const id-bytes (int 2)) @@ -53,74 +55,80 @@ ;;;; Shared low-level stream stuff -(defn- write-id! [^DataOutputStream stream ^Integer id] (.writeByte stream id)) +(defn- write-id [^DataOutputStream stream ^Integer id] (.writeByte stream id)) -(defn- write-bytes! +(defn- write-bytes "Writes arbitrary byte data, preceded by its length." [^DataOutputStream stream ^bytes ba] (let [size (alength ba)] (.writeInt stream size) ; Encode size of byte array (.write stream ba 0 size))) -(defn- write-biginteger! - "Wrapper around `write-bytes!` for common case of writing a BigInteger." +(defn- write-biginteger + "Wrapper around `write-bytes` for common case of writing a BigInteger." [^DataOutputStream stream ^BigInteger x] - (write-bytes! stream (.toByteArray x))) + (write-bytes stream (.toByteArray x))) -(defn- read-bytes! +(defn- read-bytes "Reads arbitrary byte data, preceded by its length." ^bytes [^DataInputStream stream] (let [size (.readInt stream) ba (byte-array size)] (.read stream ba 0 size) ba)) -(defn- read-biginteger! - "Wrapper around `read-bytes!` for common case of reading a BigInteger. +(defn- read-biginteger + "Wrapper around `read-bytes` for common case of reading a BigInteger. Note that as of Clojure 1.3, java.math.BigInteger ≠ clojure.lang.BigInt." ^BigInteger [^DataInputStream stream] - (BigInteger. (read-bytes! stream))) + (BigInteger. (read-bytes stream))) ;;;; Freezing -(defprotocol Freezable (freeze [this stream])) +(defprotocol Freezable (freeze-to-stream* [this stream])) -(defmacro freezer +(defn- freeze-to-stream + "Like `freeze-to-stream*` but with metadata support." + [x ^DataOutputStream s] + (if-let [m (meta x)] + (do (write-id s id-meta) + (freeze-to-stream m s))) + (freeze-to-stream* x s)) + +(defmacro ^:private freezer "Helper to extend Freezable protocol." [type id & body] `(extend-type ~type ~'Freezable - (~'freeze [~'x ~(with-meta 's {:tag 'DataOutputStream})] - (write-id! ~'s ~id) + (~'freeze-to-stream* [~'x ~(with-meta 's {:tag 'DataOutputStream})] + (write-id ~'s ~id) ~@body))) -(defmacro coll-freezer +(defmacro ^:private coll-freezer "Extends Freezable to simple collection types." [type id & body] `(freezer ~type ~id - (.writeInt ~'s (count ~'x)) ; Encode collection length - (doseq [i# ~'x] (freeze-to-stream!* ~'s i#)))) + (.writeInt ~'s (count ~'x)) + (doseq [i# ~'x] (freeze-to-stream i# ~'s)))) -(defmacro kv-freezer +(defmacro ^:private kv-freezer "Extends Freezable to key-value collection types." [type id & body] `(freezer ~type ~id - (.writeInt ~'s (* 2 (count ~'x))) ; Encode num kvs + (.writeInt ~'s (* 2 (count ~'x))) (doseq [[k# v#] ~'x] - (freeze-to-stream!* ~'s k#) - (freeze-to-stream!* ~'s v#)))) + (freeze-to-stream k# ~'s) + (freeze-to-stream v# ~'s)))) -(freezer (Class/forName "[B") id-bytes (write-bytes! s x)) +(freezer (Class/forName "[B") id-bytes (write-bytes s x)) (freezer nil id-nil) (freezer Boolean id-boolean (.writeBoolean s x)) (freezer Character id-char (.writeChar s (int x))) -(freezer String id-string (write-bytes! s (.getBytes x "UTF-8"))) +(freezer String id-string (write-bytes s (.getBytes x "UTF-8"))) (freezer Keyword id-keyword (.writeUTF s (if-let [ns (namespace x)] (str ns "/" (name x)) (name x)))) -(declare freeze-to-stream!*) - (coll-freezer PersistentQueue id-queue) (coll-freezer PersistentTreeSet id-sorted-set) (kv-freezer PersistentTreeMap id-sorted-map) @@ -135,148 +143,90 @@ (freezer Short id-short (.writeShort s x)) (freezer Integer id-integer (.writeInt s x)) (freezer Long id-long (.writeLong s x)) -(freezer BigInt id-bigint (write-biginteger! s (.toBigInteger x))) -(freezer BigInteger id-bigint (write-biginteger! s x)) +(freezer BigInt id-bigint (write-biginteger s (.toBigInteger x))) +(freezer BigInteger id-bigint (write-biginteger s x)) (freezer Float id-float (.writeFloat s x)) (freezer Double id-double (.writeDouble s x)) (freezer BigDecimal id-bigdec - (write-biginteger! s (.unscaledValue x)) + (write-biginteger s (.unscaledValue x)) (.writeInt s (.scale x))) (freezer Ratio id-ratio - (write-biginteger! s (.numerator x)) - (write-biginteger! s (.denominator x))) + (write-biginteger s (.numerator x)) + (write-biginteger s (.denominator x))) ;; Use Clojure's own reader as final fallback -(freezer Object id-reader (write-bytes! s (.getBytes (pr-str x) "UTF-8"))) +(freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8"))) -(defn- freeze-to-stream!* [^DataOutputStream s x] - (if-let [m (meta x)] - (do (write-id! s id-meta) - (freeze-to-stream!* s m))) - (freeze x s)) - -(defn- freeze-to-stream! - "Serializes x to given output stream." - ([data-output-stream x] ; For <= 1.0.1 compatibility - (freeze-to-stream! data-output-stream x true)) - ([data-output-stream x print-dup?] - (binding [*print-dup* print-dup?] ; For `pr-str` - (freeze-to-stream!* data-output-stream x)))) - -(defn freeze-to-bytes - "Serializes x to a byte array and returns the array." - ^bytes [x & {:keys [compress? print-dup? password] - :or {compress? true - print-dup? true}}] - (let [ba (ByteArrayOutputStream.) - stream (DataOutputStream. ba)] - (freeze-to-stream! stream x print-dup?) - (let [ba (.toByteArray ba) - ba (if compress? (utils/compress-snappy ba) ba) - ba (if password (crypto/encrypt-aes128 password ba) ba)] - ba))) +;; TODO New `freeze` API ;;;; Thawing -(declare thaw-from-stream!*) +(declare thaw-from-stream) -(defn coll-thaw! +(defn coll-thaw "Thaws simple collection types." [^DataInputStream s] - (repeatedly (.readInt s) #(thaw-from-stream!* s))) + (repeatedly (.readInt s) #(thaw-from-stream s))) -(defn coll-thaw-kvs! +(defn coll-thaw-kvs "Thaws key-value collection types." [^DataInputStream s] (repeatedly (/ (.readInt s) 2) - (fn [] [(thaw-from-stream!* s) (thaw-from-stream!* s)]))) + (fn [] [(thaw-from-stream s) (thaw-from-stream s)]))) -(defn- thaw-from-stream!* +(defn- thaw-from-stream [^DataInputStream s] (let [type-id (.readByte s)] (utils/case-eval type-id - id-reader (read-string (String. (read-bytes! s) "UTF-8")) - id-bytes (read-bytes! s) + id-reader (read-string (String. (read-bytes s) "UTF-8")) + id-bytes (read-bytes s) id-nil nil id-boolean (.readBoolean s) id-char (.readChar s) - id-string (String. (read-bytes! s) "UTF-8") + id-string (String. (read-bytes s) "UTF-8") id-keyword (keyword (.readUTF s)) - id-queue (into (PersistentQueue/EMPTY) (coll-thaw! s)) - id-sorted-set (into (sorted-set) (coll-thaw! s)) - id-sorted-map (into (sorted-map) (coll-thaw-kvs! s)) + id-queue (into (PersistentQueue/EMPTY) (coll-thaw s)) + id-sorted-set (into (sorted-set) (coll-thaw s)) + id-sorted-map (into (sorted-map) (coll-thaw-kvs s)) - id-list (into '() (reverse (coll-thaw! s))) - id-vector (into [] (coll-thaw! s)) - id-set (into #{} (coll-thaw! s)) - id-map (into {} (coll-thaw-kvs! s)) - id-coll (doall (coll-thaw! s)) + id-list (into '() (reverse (coll-thaw s))) + id-vector (into [] (coll-thaw s)) + id-set (into #{} (coll-thaw s)) + id-map (into {} (coll-thaw-kvs s)) + id-coll (doall (coll-thaw s)) - id-meta (let [m (thaw-from-stream!* s)] (with-meta (thaw-from-stream!* s) m)) + id-meta (let [m (thaw-from-stream s)] (with-meta (thaw-from-stream s) m)) id-byte (.readByte s) id-short (.readShort s) id-integer (.readInt s) id-long (.readLong s) - id-bigint (bigint (read-biginteger! s)) + id-bigint (bigint (read-biginteger s)) id-float (.readFloat s) id-double (.readDouble s) - id-bigdec (BigDecimal. (read-biginteger! s) (.readInt s)) + id-bigdec (BigDecimal. (read-biginteger s) (.readInt s)) - id-ratio (/ (bigint (read-biginteger! s)) - (bigint (read-biginteger! s))) + id-ratio (/ (bigint (read-biginteger s)) + (bigint (read-biginteger s))) ;;; DEPRECATED id-old-reader (read-string (.readUTF s)) id-old-string (.readUTF s) id-old-map (apply hash-map (repeatedly (* 2 (.readInt s)) - #(thaw-from-stream!* s))) + #(thaw-from-stream s))) (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) -(defn- thaw-from-stream! - "Deserializes an object from given input stream." - [data-input-stream read-eval?] - (binding [*read-eval* read-eval?] - (let [;; Support older versions of Nippy that wrote a version header - maybe-schema-header (thaw-from-stream!* data-input-stream)] - (if (and (string? maybe-schema-header) - (.startsWith ^String maybe-schema-header "\u0000~")) - (thaw-from-stream!* data-input-stream) - maybe-schema-header)))) +;; TODO New `thaw` API -(defn thaw-from-bytes - "Deserializes an object from given byte array." - [ba & {:keys [compressed? read-eval? password] - :or {compressed? true - read-eval? false ; For `read-string` injection safety - NB!!! - }}] - (try - (-> (let [ba (if password (crypto/decrypt-aes128 password ba) ba) - ba (if compressed? (utils/uncompress-snappy ba) ba)] - ba) - (ByteArrayInputStream.) - (DataInputStream.) - (thaw-from-stream! read-eval?)) - (catch Exception e - (throw (Exception. - (cond password "Thaw failed. Unencrypted data or bad password?" - compressed? "Thaw failed. Encrypted or uncompressed data?" - :else "Thaw failed. Encrypted and/or compressed data?") - e))))) - -(comment - (-> (freeze-to-bytes "my data" :password [:salted "password"]) - (thaw-from-bytes)) - (-> (freeze-to-bytes "my data" :compress? true) - (thaw-from-bytes :compressed? false))) +;;;; Stress data (def stress-data "Reference data used for tests & benchmarks." (let [] @@ -322,4 +272,64 @@ ;; Clojure 1.4+ tagged literals :tagged-uuid (java.util.UUID/randomUUID) - :tagged-date (java.util.Date.)})) \ No newline at end of file + :tagged-date (java.util.Date.)})) + +;;;; Deprecated API + +(defn- freeze-to-stream-outer + "Serializes x to given output stream." + ([data-output-stream x] ; For <= 1.0.1 compatibility + (freeze-to-stream-outer data-output-stream x true)) + ([data-output-stream x print-dup?] + (binding [*print-dup* print-dup?] ; For `pr-str` + (freeze-to-stream x data-output-stream)))) + +(defn freeze-to-bytes + "Serializes x to a byte array and returns the array." + ^bytes [x & {:keys [compress? print-dup? password] + :or {compress? true + print-dup? true}}] + (let [ba (ByteArrayOutputStream.) + stream (DataOutputStream. ba)] + (freeze-to-stream-outer stream x print-dup?) + (let [ba (.toByteArray ba) + ba (if compress? (utils/compress-snappy ba) ba) + ba (if password (crypto/encrypt-aes128 password ba) ba)] + ba))) + +(defn- thaw-from-stream-outer + "Deserializes an object from given input stream." + [data-input-stream read-eval?] + (binding [*read-eval* read-eval?] + (let [;; Support older versions of Nippy that wrote a version header + maybe-schema-header (thaw-from-stream data-input-stream)] + (if (and (string? maybe-schema-header) + (.startsWith ^String maybe-schema-header "\u0000~")) + (thaw-from-stream data-input-stream) + maybe-schema-header)))) + +(defn thaw-from-bytes + "Deserializes an object from given byte array." + [ba & {:keys [compressed? read-eval? password] + :or {compressed? true + read-eval? false ; For `read-string` injection safety - NB!!! + }}] + (try + (-> (let [ba (if password (crypto/decrypt-aes128 password ba) ba) + ba (if compressed? (utils/uncompress-snappy ba) ba)] + ba) + (ByteArrayInputStream.) + (DataInputStream.) + (thaw-from-stream-outer read-eval?)) + (catch Exception e + (throw (Exception. + (cond password "Thaw failed. Unencrypted data or bad password?" + compressed? "Thaw failed. Encrypted or uncompressed data?" + :else "Thaw failed. Encrypted and/or compressed data?") + e))))) + +(comment + (-> (freeze-to-bytes "my data" :password [:salted "password"]) + (thaw-from-bytes)) + (-> (freeze-to-bytes "my data" :compress? true) + (thaw-from-bytes :compressed? false))) \ No newline at end of file From 9734e882bb350dcba0762ca49a2dfac49c02fabb Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 01:17:22 +0700 Subject: [PATCH 06/14] Swap `repeatedly` -> `utils/repeatedly*` (faster) --- src/taoensso/nippy.clj | 12 +++++++----- src/taoensso/nippy/utils.clj | 7 +++++++ test/taoensso/nippy/tests/main.clj | 4 ++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 5191513..781c11b 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -168,13 +168,13 @@ (defn coll-thaw "Thaws simple collection types." [^DataInputStream s] - (repeatedly (.readInt s) #(thaw-from-stream s))) + (utils/repeatedly* (.readInt s) #(thaw-from-stream s))) (defn coll-thaw-kvs "Thaws key-value collection types." [^DataInputStream s] - (repeatedly (/ (.readInt s) 2) - (fn [] [(thaw-from-stream s) (thaw-from-stream s)]))) + (utils/repeatedly* (/ (.readInt s) 2) + (fn [] [(thaw-from-stream s) (thaw-from-stream s)]))) (defn- thaw-from-stream [^DataInputStream s] @@ -195,7 +195,9 @@ id-sorted-set (into (sorted-set) (coll-thaw s)) id-sorted-map (into (sorted-map) (coll-thaw-kvs s)) - id-list (into '() (reverse (coll-thaw s))) + ;;id-list (into '() (reverse (coll-thaw s))) + ;;id-vector (into [] (coll-thaw s)) + id-list (into '() (rseq (coll-thaw s))) id-vector (into [] (coll-thaw s)) id-set (into #{} (coll-thaw s)) id-map (into {} (coll-thaw-kvs s)) @@ -219,7 +221,7 @@ ;;; DEPRECATED id-old-reader (read-string (.readUTF s)) id-old-string (.readUTF s) - id-old-map (apply hash-map (repeatedly (* 2 (.readInt s)) + id-old-map (apply hash-map (utils/repeatedly* (* 2 (.readInt s)) #(thaw-from-stream s))) (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index c3eeeae..ecdcc65 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -13,6 +13,13 @@ clauses) ~(when default default)))) +(defn repeatedly* "Like `repeatedly` but faster and returns a vector." + [n f] + (loop [v (transient []) idx 0] + (if (>= idx n) + (persistent! v) + (recur (conj! v (f)) (inc idx))))) + (defmacro time-ns "Returns number of nanoseconds it takes to execute body." [& body] `(let [t0# (System/nanoTime)] ~@body (- (System/nanoTime) t0#))) diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 47625ca..4fb2b09 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -10,9 +10,9 @@ (def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"]) #(nippy/freeze-to-bytes % :password [:cached "secret"]))) -(expect test-data (roundtrip-defaults test-data)) +(expect-focused test-data (roundtrip-defaults test-data)) ; TODO (expect test-data (roundtrip-encrypted test-data)) -(expect ; Snappy lib compatibility +#_(expect ; Snappy lib compatibility ; TODO (let [thaw #(nippy/thaw-from-bytes % :compressed? false) ^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false) ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) From 613c77b8a1bcec389147534b2c7b976eb71e0fba Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 01:33:32 +0700 Subject: [PATCH 07/14] Further prep for new API, mark `freeze-to-bytes` and `thaw-from-bytes` as deprecated --- src/taoensso/nippy.clj | 45 +++++++++++------------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 781c11b..0007207 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -278,51 +278,29 @@ ;;;; Deprecated API -(defn- freeze-to-stream-outer - "Serializes x to given output stream." - ([data-output-stream x] ; For <= 1.0.1 compatibility - (freeze-to-stream-outer data-output-stream x true)) - ([data-output-stream x print-dup?] - (binding [*print-dup* print-dup?] ; For `pr-str` - (freeze-to-stream x data-output-stream)))) - -(defn freeze-to-bytes - "Serializes x to a byte array and returns the array." +;; TODO Rewrite in :legacy terms +(defn freeze-to-bytes "DEPRECATED: Use `freeze` instead." ^bytes [x & {:keys [compress? print-dup? password] :or {compress? true print-dup? true}}] (let [ba (ByteArrayOutputStream.) stream (DataOutputStream. ba)] - (freeze-to-stream-outer stream x print-dup?) + (binding [*print-dup* print-dup?] (freeze-to-stream x stream)) (let [ba (.toByteArray ba) ba (if compress? (utils/compress-snappy ba) ba) ba (if password (crypto/encrypt-aes128 password ba) ba)] ba))) -(defn- thaw-from-stream-outer - "Deserializes an object from given input stream." - [data-input-stream read-eval?] - (binding [*read-eval* read-eval?] - (let [;; Support older versions of Nippy that wrote a version header - maybe-schema-header (thaw-from-stream data-input-stream)] - (if (and (string? maybe-schema-header) - (.startsWith ^String maybe-schema-header "\u0000~")) - (thaw-from-stream data-input-stream) - maybe-schema-header)))) - -(defn thaw-from-bytes - "Deserializes an object from given byte array." +;; TODO Rewrite in :legacy terms +(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead." [ba & {:keys [compressed? read-eval? password] - :or {compressed? true - read-eval? false ; For `read-string` injection safety - NB!!! - }}] + :or {read-eval? false ; For `read-string` injection safety - NB!!! + compressed? true}}] (try - (-> (let [ba (if password (crypto/decrypt-aes128 password ba) ba) - ba (if compressed? (utils/uncompress-snappy ba) ba)] - ba) - (ByteArrayInputStream.) - (DataInputStream.) - (thaw-from-stream-outer read-eval?)) + (let [ba (if password (crypto/decrypt-aes128 password ba) ba) + ba (if compressed? (utils/uncompress-snappy ba) ba) + stream (DataInputStream. (ByteArrayInputStream. ba))] + (binding [*read-eval* read-eval?] (thaw-from-stream stream))) (catch Exception e (throw (Exception. (cond password "Thaw failed. Unencrypted data or bad password?" @@ -331,6 +309,7 @@ e))))) (comment + ;; Errors (-> (freeze-to-bytes "my data" :password [:salted "password"]) (thaw-from-bytes)) (-> (freeze-to-bytes "my data" :compress? true) From 56a97d240e36613db12ec3009b98fe857c77d413 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 11:54:08 +0700 Subject: [PATCH 08/14] Update README for new API --- README.md | 24 +++++++++++++----------- project.clj | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8f2da43..c1d6dc4 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,16 @@ Current [semantic](http://semver.org/) version: ```clojure [com.taoensso/nippy "1.2.1"] ; Stable -[com.taoensso/nippy "1.3.0-alpha3"] ; Development (adds crypto support!) +[com.taoensso/nippy "2.0.0-alpha1"] ; Development (see notes below) ``` +2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`. + # Nippy, a Clojure serialization library Clojure's [rich data types](http://clojure.org/datatypes) are *awesome*. And its [reader](http://clojure.org/reader) allows you to take your data just about anywhere. But the reader can be painfully slow when you've got a lot of data to crunch (like when you're serializing to a database). -Nippy is an attempt to provide a drop-in, high-performance alternative to the reader. It's a fork of [Deep-Freeze](https://github.com/halgari/deep-freeze) and is used as the [Carmine Redis client](https://github.com/ptaoussanis/carmine) serializer. +Nippy is an attempt to provide a reliable, high-performance **drop-in alternative to the reader**. It's used, among others, as the [Carmine Redis client](https://github.com/ptaoussanis/carmine) and [Faraday DynamoDB client]https://github.com/ptaoussanis/faraday) serializer. ## What's in the box™? * Small, uncomplicated **all-Clojure** library. @@ -17,8 +19,8 @@ Nippy is an attempt to provide a drop-in, high-performance alternative to the re * Comprehesive, extensible **support for all major data types**. * **Reader-fallback** for difficult/future types (including Clojure 1.4+ tagged literals). * **Full test coverage** for every supported type. - * [Snappy](http://code.google.com/p/snappy/) **integrated de/compression** for efficient storage and network transfer. - * Enable **high-strength encryption** with a single `:password [:salted "my-password"]` option. (1.3.0+) + * Fully pluggable **compression**, including built-in high-performance [Snappy](http://code.google.com/p/snappy/) compressor. + * Fully pluggable **encryption**, including built-in high-strength AES128 enabled with a single `:password [:salted "my-password"]` option. (2.0.0+) ## Getting started @@ -83,14 +85,14 @@ nippy/stress-data Serialize it: ```clojure -(def frozen-stress-data (nippy/freeze-to-bytes nippy/stress-data)) +(def frozen-stress-data (nippy/freeze nippy/stress-data)) => # ``` Deserialize it: ```clojure -(nippy/thaw-from-bytes frozen-stress-data) +(nippy/thaw frozen-stress-data) => {:bytes (byte-array [(byte 1) (byte 2) (byte 3)]) :nil nil :boolean true @@ -101,14 +103,14 @@ Couldn't be simpler! ### Encryption (currently in **ALPHA**) -As of 1.3.0, Nippy also gives you **dead simple data encryption**. Add a single option to your usual freeze/thaw calls like so: +As of 2.0.0, Nippy also gives you **dead simple data encryption**. Add a single option to your usual freeze/thaw calls like so: ```clojure -(nippy/freeze-to-bytes nippy/stress-data :password [:salted "my-password"]) ; Encrypt -(nippy/thaw-from-bytes :password [:salted "my-password"]) ; Decrypt +(nippy/freeze nippy/stress-data {:password [:salted "my-password"]}) ; Encrypt +(nippy/thaw {:password [:salted "my-password"]}) ; Decrypt ``` -There's two forms of encryption on offer: `:salted` and `:cached`. Each of these makes carefully-chosen trade-offs and is suited to one of two common use cases. See the `aes128-salted` and `aes128-cached` [docstrings](http://ptaoussanis.github.io/nippy/taoensso.nippy.crypto.html) for a detailed explanation of why/when you'd want one or the other. +There's two default forms of encryption on offer: `:salted` and `:cached`. Each of these makes carefully-chosen trade-offs and is suited to one of two common use cases. See the `default-aes128-encryptor` [docstring](http://ptaoussanis.github.io/nippy/taoensso.nippy.encryption.html) for a detailed explanation of why/when you'd want one or the other. ## Performance @@ -135,4 +137,4 @@ Otherwise reach me (Peter Taoussanis) at [taoensso.com](https://www.taoensso.com ## License -Copyright © 2012, 2013 Peter Taoussanis. Distributed under the [Eclipse Public License](http://www.eclipse.org/legal/epl-v10.html), the same as Clojure. +Copyright © 2012, 2013 Peter Taoussanis. Distributed under the [Eclipse Public License](http://www.eclipse.org/legal/epl-v10.html), the same as Clojure. \ No newline at end of file diff --git a/project.clj b/project.clj index 7451f11..e3985b4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "1.3.0-alpha3" +(defproject com.taoensso/nippy "2.0.0-alpha1" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License" From e5a9e1f671f818ad79e3fe5b04c716ba150b7e34 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 12:00:43 +0700 Subject: [PATCH 09/14] Add lein :bench profile & alias --- project.clj | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/project.clj b/project.clj index e3985b4..7f34a34 100644 --- a/project.clj +++ b/project.clj @@ -6,13 +6,16 @@ :dependencies [[org.clojure/clojure "1.4.0"] [expectations "1.4.43"] [org.iq80.snappy/snappy "0.3"]] - :profiles {:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} - :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} - :dev {:dependencies []} - :test {:dependencies [[org.xerial.snappy/snappy-java "1.0.5-M3"]]}} - :aliases {"test-all" ["with-profile" "test,1.4:test,1.5" "expectations"] - "test-auto" ["with-profile" "test" "autoexpect"] - "start-dev" ["with-profile" "dev,test" "repl" ":headless"]} + :profiles {:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} + :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} + :dev {:dependencies []} + :test {:dependencies [[org.xerial.snappy/snappy-java "1.0.5-M3"]]} + :bench {:dependencies [] + :jvm-opts ["-server" "-XX:+UseCompressedOops"]}} + :aliases {"test-all" ["with-profile" "test,1.4:test,1.5" "expectations"] + "test-auto" ["with-profile" "test" "autoexpect"] + "start-dev" ["with-profile" "dev,test,bench" "repl" ":headless"] + "start-bench" ["trampoline" "start-dev"]} :plugins [[lein-expectations "0.0.7"] [lein-autoexpect "0.2.5"] [codox "0.6.4"]] From bfc16ce3ab8455f173f2302bd1ebf6d0dc920dbd Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 02:27:00 +0700 Subject: [PATCH 10/14] Optimize collection creation (much faster) --- src/taoensso/nippy.clj | 28 +++++++++++++--------------- src/taoensso/nippy/utils.clj | 18 ++++++++++++------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 0007207..dc82023 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -167,13 +167,13 @@ (defn coll-thaw "Thaws simple collection types." - [^DataInputStream s] - (utils/repeatedly* (.readInt s) #(thaw-from-stream s))) + [coll ^DataInputStream s] + (utils/repeatedly* coll (.readInt s) #(thaw-from-stream s))) (defn coll-thaw-kvs "Thaws key-value collection types." - [^DataInputStream s] - (utils/repeatedly* (/ (.readInt s) 2) + [coll ^DataInputStream s] + (utils/repeatedly* coll (/ (.readInt s) 2) (fn [] [(thaw-from-stream s) (thaw-from-stream s)]))) (defn- thaw-from-stream @@ -191,17 +191,15 @@ id-string (String. (read-bytes s) "UTF-8") id-keyword (keyword (.readUTF s)) - id-queue (into (PersistentQueue/EMPTY) (coll-thaw s)) - id-sorted-set (into (sorted-set) (coll-thaw s)) - id-sorted-map (into (sorted-map) (coll-thaw-kvs s)) + id-queue (coll-thaw (PersistentQueue/EMPTY) s) + id-sorted-set (coll-thaw (sorted-set) s) + id-sorted-map (coll-thaw-kvs (sorted-map) s) - ;;id-list (into '() (reverse (coll-thaw s))) - ;;id-vector (into [] (coll-thaw s)) - id-list (into '() (rseq (coll-thaw s))) - id-vector (into [] (coll-thaw s)) - id-set (into #{} (coll-thaw s)) - id-map (into {} (coll-thaw-kvs s)) - id-coll (doall (coll-thaw s)) + id-list (into '() (rseq (coll-thaw [] s))) + id-vector (coll-thaw [] s) + id-set (coll-thaw #{} s) + id-map (coll-thaw-kvs {} s) + id-coll (seq (coll-thaw [] s)) id-meta (let [m (thaw-from-stream s)] (with-meta (thaw-from-stream s) m)) @@ -221,7 +219,7 @@ ;;; DEPRECATED id-old-reader (read-string (.readUTF s)) id-old-string (.readUTF s) - id-old-map (apply hash-map (utils/repeatedly* (* 2 (.readInt s)) + id-old-map (apply hash-map (repeatedly (* 2 (.readInt s)) #(thaw-from-stream s))) (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index ecdcc65..28eb549 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -13,12 +13,18 @@ clauses) ~(when default default)))) -(defn repeatedly* "Like `repeatedly` but faster and returns a vector." - [n f] - (loop [v (transient []) idx 0] - (if (>= idx n) - (persistent! v) - (recur (conj! v (f)) (inc idx))))) +(defn repeatedly* + "Like `repeatedly` but faster and returns given collection type." + [coll n f] + (if-not (instance? clojure.lang.IEditableCollection coll) + (loop [v coll idx 0] + (if (>= idx n) + v + (recur (conj v (f)) (inc idx)))) + (loop [v (transient coll) idx 0] + (if (>= idx n) + (persistent! v) + (recur (conj! v (f)) (inc idx)))))) (defmacro time-ns "Returns number of nanoseconds it takes to execute body." [& body] `(let [t0# (System/nanoTime)] ~@body (- (System/nanoTime) t0#))) From 5a398efd9f8f3479f7d3801b2251fe4a890c901e Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 12:06:17 +0700 Subject: [PATCH 11/14] Rename `repeatedly*` -> `repeatedly-into` --- src/taoensso/nippy.clj | 8 ++++---- src/taoensso/nippy/utils.clj | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index dc82023..7324f2a 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -168,12 +168,12 @@ (defn coll-thaw "Thaws simple collection types." [coll ^DataInputStream s] - (utils/repeatedly* coll (.readInt s) #(thaw-from-stream s))) + (utils/repeatedly-into coll (.readInt s) #(thaw-from-stream s))) (defn coll-thaw-kvs "Thaws key-value collection types." [coll ^DataInputStream s] - (utils/repeatedly* coll (/ (.readInt s) 2) + (utils/repeatedly-into coll (/ (.readInt s) 2) (fn [] [(thaw-from-stream s) (thaw-from-stream s)]))) (defn- thaw-from-stream @@ -219,8 +219,8 @@ ;;; DEPRECATED id-old-reader (read-string (.readUTF s)) id-old-string (.readUTF s) - id-old-map (apply hash-map (repeatedly (* 2 (.readInt s)) - #(thaw-from-stream s))) + id-old-map (apply hash-map (utils/repeatedly-into [] (* 2 (.readInt s)) + #(thaw-from-stream s))) (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 28eb549..96bfcaa 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -13,8 +13,8 @@ clauses) ~(when default default)))) -(defn repeatedly* - "Like `repeatedly` but faster and returns given collection type." +(defn repeatedly-into + "Like `repeatedly` but faster and `conj`s items into given collection." [coll n f] (if-not (instance? clojure.lang.IEditableCollection coll) (loop [v coll idx 0] From 284d11c6609558fdcc8176d7060e069db4193b80 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 12:40:46 +0700 Subject: [PATCH 12/14] Prep for pluggable compression+encryption --- src/taoensso/nippy/compression.clj | 23 +++ .../nippy/{crypto.clj => encryption.clj} | 136 +++++++++--------- src/taoensso/nippy/utils.clj | 6 +- 3 files changed, 92 insertions(+), 73 deletions(-) create mode 100644 src/taoensso/nippy/compression.clj rename src/taoensso/nippy/{crypto.clj => encryption.clj} (56%) diff --git a/src/taoensso/nippy/compression.clj b/src/taoensso/nippy/compression.clj new file mode 100644 index 0000000..7c2b625 --- /dev/null +++ b/src/taoensso/nippy/compression.clj @@ -0,0 +1,23 @@ +(ns taoensso.nippy.compression + "Alpha - subject to change." + {:author "Peter Taoussanis"} + (:require [taoensso.nippy.utils :as utils])) + +;;;; Interface + +(defprotocol ICompressor + (header-id [compressor]) ; Unique, >0, <= 128 + (compress ^bytes [compressor ba]) + (decompress ^bytes [compressor ba])) + +;;;; Default implementations + +(deftype DefaultSnappyCompressor [] + ICompressor + (header-id [_] 1) + (compress [_ ba] (org.iq80.snappy.Snappy/compress ba)) + (decompress [_ ba] (org.iq80.snappy.Snappy/uncompress ba 0 (alength ^bytes ba)))) + +(def default-snappy-compressor + "Default org.iq80.snappy.Snappy compressor." + (DefaultSnappyCompressor.)) \ No newline at end of file diff --git a/src/taoensso/nippy/crypto.clj b/src/taoensso/nippy/encryption.clj similarity index 56% rename from src/taoensso/nippy/crypto.clj rename to src/taoensso/nippy/encryption.clj index f3582f3..a0b65ef 100644 --- a/src/taoensso/nippy/crypto.clj +++ b/src/taoensso/nippy/encryption.clj @@ -1,24 +1,19 @@ -(ns taoensso.nippy.crypto +(ns taoensso.nippy.encryption "Alpha - subject to change. Simple no-nonsense crypto with reasonable defaults. Because your Clojure data deserves some privacy." {:author "Peter Taoussanis"} - (:require [clojure.string :as str] - [taoensso.nippy.utils :as utils])) + (:require [taoensso.nippy.utils :as utils])) ;;;; Interface -(defprotocol IEncrypter - (gen-key ^javax.crypto.spec.SecretKeySpec [encrypter salt-ba pwd]) - (encrypt ^bytes [encrypter pwd ba]) - (decrypt ^bytes [encrypter pwd ba])) +(defprotocol IEncryptor + (header-id [encryptor]) ; Unique, >0, <= 128 + (encrypt ^bytes [encryptor pwd ba]) + (decrypt ^bytes [encryptor pwd ba])) -(defrecord AES128Encrypter [key-work-factor key-cache]) +;;;; Default digests, ciphers, etc. -;;;; Digests, ciphers, etc. - -;; 128bit keys have good JVM availability and are -;; entirely sufficient, Ref. http://goo.gl/2YRQG (def ^:private ^:const aes128-block-size (int 16)) (def ^:private ^:const salt-size (int 16)) @@ -36,10 +31,10 @@ (defn- sha512-key "SHA512-based key generator. Good JVM availability without extra dependencies (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds." - [salt-ba ^String pwd key-work-factor] + ^javax.crypto.spec.SecretKeySpec [salt-ba ^String pwd] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] (if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba)) - n (* (int Short/MAX_VALUE) key-work-factor)] + n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] (if-not (zero? n) (recur (.digest sha512-md ba) (dec n)) (-> ba (java.util.Arrays/copyOf aes128-block-size) @@ -52,37 +47,60 @@ (time (sha512-key nil "hi" 128)) ; ~4500ms (paranoid) ) -;;;; Default implementation +;;;; Default implementations -(extend-type AES128Encrypter - IEncrypter - (gen-key [{:keys [key-work-factor key-cache]} salt-ba pwd] - ;; Trade-off: salt-ba and key-cache mutually exclusive - (utils/memoized key-cache sha512-key salt-ba pwd key-work-factor)) +(defn- destructure-typed-pwd + [typed-password] + (letfn [(throw-ex [] + (throw (Exception. + (str "Expected password form: " + "[<#{:salted :cached}> ].\n " + "See `default-aes128-encryptor` docstring for details!"))))] + (if-not (vector? typed-password) + (throw-ex) + (let [[type password] typed-password] + (if-not (#{:salted :cached} type) + (throw-ex) + [type password]))))) - (encrypt [{:keys [key-cache] :as this} pwd data-ba] - (let [salt? (not key-cache) - iv-ba (rand-bytes aes128-block-size) - salt-ba (when salt? (rand-bytes salt-size)) - prefix-ba (if-not salt? iv-ba (utils/ba-concat iv-ba salt-ba)) - key (gen-key this salt-ba pwd) - iv (javax.crypto.spec.IvParameterSpec. iv-ba)] +(comment (destructure-typed-pwd [:salted "foo"])) + +(defrecord DefaultAES128Encryptor [key-cache] + IEncryptor + (header-id [_] 1) + + (encrypt [this typed-pwd data-ba] + (let [[type pwd] (destructure-typed-pwd typed-pwd) + salt? (= type :salted) + iv-ba (rand-bytes aes128-block-size) + salt-ba (when salt? (rand-bytes salt-size)) + prefix-ba (if-not salt? iv-ba (utils/ba-concat iv-ba salt-ba)) + key (utils/memoized (when-not salt? (:key-cache this)) + sha512-key salt-ba pwd) + iv (javax.crypto.spec.IvParameterSpec. iv-ba)] (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE key iv) (utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba)))) - (decrypt [{:keys [key-cache] :as this} pwd ba] - (let [salt? (not key-cache) + (decrypt [this typed-pwd ba] + (let [[type pwd] (destructure-typed-pwd typed-pwd) + salt? (= type :salted) prefix-size (+ aes128-block-size (if salt? salt-size 0)) [prefix-ba data-ba] (utils/ba-split ba prefix-size) [iv-ba salt-ba] (if-not salt? [prefix-ba nil] (utils/ba-split prefix-ba aes128-block-size)) - key (gen-key this salt-ba pwd) + key (utils/memoized (when-not salt? (:key-cache this)) + sha512-key salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE key iv) (.doFinal aes128-cipher data-ba)))) -(def aes128-salted - "USE CASE: You want more than a small, finite number of passwords (e.g. each +(def default-aes128-encryptor + "Alpha - subject to change. + Default 128bit AES encryptor with multi-round SHA-512 keygen. + + Password form [:salted \"my-password\"] + --------------------------------------- + USE CASE: You want more than a small, finite number of passwords (e.g. each item encrypted will use a unique user-provided password). IMPLEMENTATION: Uses a relatively cheap key hash, but automatically salts @@ -94,13 +112,13 @@ particular key. Slower than `aes128-cached`, and easier to attack any particular key - but - keys are independent." - (AES128Encrypter. 5 nil)) + keys are independent. -(def aes128-cached - "USE CASE: You want only a small, finite number of passwords (e.g. a limited - number of staff/admins, or you'll be using a single password to - encrypt many items). + Password form [:cached \"my-password\"] + --------------------------------------- + USE CASE: You want only a small, finite number of passwords (e.g. a limited + number of staff/admins, or you'll be using a single password to + encrypt many items). IMPLEMENTATION: Uses a _very_ expensive (but cached) key hash, and no salt. @@ -112,37 +130,19 @@ Faster than `aes128-salted`, and harder to attack any particular key - but increased danger if a key is somehow compromised." - (AES128Encrypter. 64 (atom {}))) + (DefaultAES128Encryptor. (atom {}))) -(defn- destructure-typed-password - "[ ] -> [Encrypter ]" - [typed-password] - (letfn [(throw-ex [] - (throw (Exception. - (str "Expected password form: " - "[<#{:salted :cached}> ].\n " - "See `aes128-salted`, `aes128-cached` for details."))))] - (if-not (vector? typed-password) - (throw-ex) - (let [[type password] typed-password] - [(case type :salted aes128-salted :cached aes128-cached (throw-ex)) - password])))) - -(defn encrypt-aes128 [typed-password ba] - (let [[encrypter password] (destructure-typed-password typed-password)] - (encrypt encrypter password ba))) - -(defn decrypt-aes128 [typed-password ba] - (let [[encrypter password] (destructure-typed-password typed-password)] - (decrypt encrypter password ba))) +;;;; Default implementation (comment - (encrypt-aes128 "my-password" (.getBytes "Secret message")) ; Malformed - (time (gen-key aes128-salted nil "my-password")) - (time (gen-key aes128-cached nil "my-password")) - (time (->> (.getBytes "Secret message" "UTF-8") - (encrypt-aes128 [:salted "p"]) - (encrypt-aes128 [:cached "p"]) - (decrypt-aes128 [:cached "p"]) - (decrypt-aes128 [:salted "p"]) + (def dae default-aes128-encryptor) + (def secret-ba (.getBytes "Secret message" "UTF-8")) + (encrypt dae "p" secret-ba) ; Malformed + (time (encrypt dae [:salted "p"] secret-ba)) + (time (encrypt dae [:cached "p"] secret-ba)) + (time (->> secret-ba + (encrypt dae [:salted "p"]) + (encrypt dae [:cached "p"]) + (decrypt dae [:cached "p"]) + (decrypt dae [:salted "p"]) (String.)))) \ No newline at end of file diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 96bfcaa..d4d1910 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -69,10 +69,6 @@ (comment (memoized nil +) (memoized nil + 5 12)) -(defn compress-snappy ^bytes [^bytes ba] (org.iq80.snappy.Snappy/compress ba)) -(defn uncompress-snappy ^bytes [^bytes ba] (org.iq80.snappy.Snappy/uncompress ba - 0 (alength ba))) - (defn ba-concat ^bytes [^bytes ba1 ^bytes ba2] (let [s1 (alength ba1) s2 (alength ba2) @@ -87,4 +83,4 @@ (comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar"))) (let [[x y] (ba-split (.getBytes "foobar") 3)] - [(String. x) (String. y)])) + [(String. x) (String. y)])) \ No newline at end of file From 8d48ec9d752650ed34c33f49b08cad429ef0b621 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 12:12:28 +0700 Subject: [PATCH 13/14] NB: Simpler, more flexible API (backwards-compatible) --- src/taoensso/nippy.clj | 167 ++++++++++++++++++++++------- src/taoensso/nippy/benchmarks.clj | 56 +++++----- src/taoensso/nippy/encryption.clj | 8 +- test/taoensso/nippy/tests/main.clj | 59 +++++++--- 4 files changed, 201 insertions(+), 89 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 7324f2a..ba54b48 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -1,16 +1,28 @@ (ns taoensso.nippy - "Simple, high-performance Clojure serialization library. Adapted from - Deep-Freeze." + "Simple, high-performance Clojure serialization library. Originally adapted + from Deep-Freeze." {:author "Peter Taoussanis"} - (:require [taoensso.nippy.utils :as utils] - [taoensso.nippy.crypto :as crypto]) + (:require [taoensso.nippy + (utils :as utils) + (compression :as compression) + (encryption :as encryption)]) (:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream ByteArrayInputStream] [clojure.lang Keyword BigInt Ratio PersistentQueue PersistentTreeMap PersistentTreeSet IPersistentList IPersistentVector IPersistentMap IPersistentSet IPersistentCollection])) -;;;; Header IDs ; TODO +;; TODO Allow ba or wrapped-ba input? +;; TODO Provide ToFreeze, Frozen, Encrypted, etc. tooling helpers + +;;;; Header IDs +;; Nippy 2.x+ prefixes frozen data with a 5-byte header: + +(def ^:const id-nippy-magic-prefix (byte 17)) +(def ^:const id-nippy-header-ver (byte 0)) +;; * Compressor id (0 if no compressor) +;; * Encryptor id (0 if no encryptor) +(def ^:const id-nippy-reserved (byte 0)) ;;;; Data type IDs @@ -159,7 +171,29 @@ ;; Use Clojure's own reader as final fallback (freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8"))) -;; TODO New `freeze` API +(defn- wrap-nippy-header [data-ba compressor encryptor password] + (let [header-ba (byte-array + [id-nippy-magic-prefix + id-nippy-header-ver + (byte (if compressor (compression/header-id compressor) 0)) + (byte (if password (encryption/header-id encryptor) 0)) + id-nippy-reserved])] + (utils/ba-concat header-ba data-ba))) + +(defn freeze + "Serializes arg (any Clojure data type) to a byte array. Enable + `:legacy-mode?` flag to produce bytes readable by Nippy < 2.x." + ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode?] + :or {print-dup? true + compressor compression/default-snappy-compressor + encryptor encryption/default-aes128-encryptor}}]] + (let [ba (ByteArrayOutputStream.) + stream (DataOutputStream. ba)] + (binding [*print-dup* print-dup?] (freeze-to-stream x stream)) + (let [ba (.toByteArray ba) + ba (if compressor (compression/compress compressor ba) ba) + ba (if password (encryption/encrypt encryptor password ba) ba)] + (if legacy-mode? ba (wrap-nippy-header ba compressor encryptor password))))) ;;;; Thawing @@ -224,7 +258,80 @@ (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) -;; TODO New `thaw` API +(defn thaw + "Deserializes frozen bytes to their original Clojure data type. Enable + `:legacy-mode?` to read bytes written by Nippy < 2.x. + + WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless + you are sure you know what you're doing." + [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode? + strict?] + :or {compressor compression/default-snappy-compressor + encryptor encryption/default-aes128-encryptor}}]] + + (let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed. " msg) e))) + thaw-data (fn [data-ba compressor password] + (let [ba data-ba + ba (if password (encryption/decrypt encryptor password ba) ba) + ba (if compressor (compression/decompress compressor ba) ba) + stream (DataInputStream. (ByteArrayInputStream. ba))] + (binding [*read-eval* read-eval?] (thaw-from-stream stream))))] + + (if legacy-mode? ; Nippy < 2.x + (try (thaw-data ba compressor password) + (catch Exception e + (cond password (ex "Unencrypted data or wrong password?" e) + compressor (ex "Encrypted or uncompressed data?" e) + :else (ex "Encrypted and/or compressed data?" e)))) + + ;; Nippy >= 2.x, we have a header! + (let [[[id-magic* id-header* id-comp* id-enc* _] data-ba] + (utils/ba-split ba 5) + + compressed? (not (zero? id-comp*)) + encrypted? (not (zero? id-enc*))] + + (cond + (not= id-magic* id-nippy-magic-prefix) + (ex (str "Not Nippy data, data frozen with Nippy < 2.x, " + "or data may be corrupt?\n" + "Enable `:legacy-mode?` option for data frozen with Nippy < 2.x.")) + + (> id-header* id-nippy-header-ver) + (ex "Data frozen with newer Nippy version. Please upgrade.") + + (and strict? (not encrypted?) password) + (ex (str "Data is not encrypted. Try again w/o password.\n" + "Disable `:strict?` option to ignore this error. ")) + + (and strict? (not compressed?) compressor) + (ex (str "Data is not compressed. Try again w/o compressor.\n" + "Disable `:strict?` option to ignore this error.")) + + (and encrypted? (not password)) + (ex "Data is encrypted. Please try again with a password.") + + (and encrypted? password + (not= id-enc* (encryption/header-id encryptor))) + (ex "Data encrypted with a different Encrypter.") + + (and compressed? compressor + (not= id-comp* (compression/header-id compressor))) + (ex "Data compressed with a different Compressor.") + + :else + (try (thaw-data data-ba (when compressed? compressor) + (when encrypted? password)) + (catch Exception e + (if (and encrypted? password) + (ex "Wrong password, or data may be corrupt?" e) + (ex "Data may be corrupt?" e))))))))) + +(comment (thaw (freeze "hello")) + (thaw (freeze "hello" {:compressor nil})) + (thaw (freeze "hello" {:compressor nil}) {:strict? true}) ; ex + (thaw (freeze "hello" {:password [:salted "p"]})) ; ex + (thaw (freeze "hello") {:password [:salted "p"]})) ;;;; Stress data @@ -276,39 +383,19 @@ ;;;; Deprecated API -;; TODO Rewrite in :legacy terms (defn freeze-to-bytes "DEPRECATED: Use `freeze` instead." - ^bytes [x & {:keys [compress? print-dup? password] - :or {compress? true - print-dup? true}}] - (let [ba (ByteArrayOutputStream.) - stream (DataOutputStream. ba)] - (binding [*print-dup* print-dup?] (freeze-to-stream x stream)) - (let [ba (.toByteArray ba) - ba (if compress? (utils/compress-snappy ba) ba) - ba (if password (crypto/encrypt-aes128 password ba) ba)] - ba))) + ^bytes [x & {:keys [print-dup? compress? password] + :or {print-dup? true + compress? true}}] + (freeze x {:print-dup? print-dup? + :compressor (when compress? compression/default-snappy-compressor) + :password password + :legacy-mode? true})) -;; TODO Rewrite in :legacy terms (defn thaw-from-bytes "DEPRECATED: Use `thaw` instead." - [ba & {:keys [compressed? read-eval? password] - :or {read-eval? false ; For `read-string` injection safety - NB!!! - compressed? true}}] - (try - (let [ba (if password (crypto/decrypt-aes128 password ba) ba) - ba (if compressed? (utils/uncompress-snappy ba) ba) - stream (DataInputStream. (ByteArrayInputStream. ba))] - (binding [*read-eval* read-eval?] (thaw-from-stream stream))) - (catch Exception e - (throw (Exception. - (cond password "Thaw failed. Unencrypted data or bad password?" - compressed? "Thaw failed. Encrypted or uncompressed data?" - :else "Thaw failed. Encrypted and/or compressed data?") - e))))) - -(comment - ;; Errors - (-> (freeze-to-bytes "my data" :password [:salted "password"]) - (thaw-from-bytes)) - (-> (freeze-to-bytes "my data" :compress? true) - (thaw-from-bytes :compressed? false))) \ No newline at end of file + [ba & {:keys [read-eval? compressed? password] + :or {compressed? true}}] + (thaw ba {:read-eval? read-eval? + :compressor (when compressed? compression/default-snappy-compressor) + :password password + :legacy-mode? true})) \ No newline at end of file diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index d004c36..d56109c 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,6 +1,6 @@ (ns taoensso.nippy.benchmarks {:author "Peter Taoussanis"} - (:require [taoensso.nippy :as nippy :refer (freeze-to-bytes thaw-from-bytes)] + (:require [taoensso.nippy :as nippy :refer (freeze thaw)] [taoensso.nippy.utils :as utils])) ;; Remove stuff from stress-data that breaks reader @@ -8,15 +8,15 @@ (defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 2000)) -(defn reader-freeze [x] (binding [*print-dup* false] (pr-str x))) -(defn reader-thaw [x] (binding [*read-eval* false] (read-string x))) -(def reader-roundtrip (comp reader-thaw reader-freeze)) +(defn freeze-reader [x] (binding [*print-dup* false] (pr-str x))) +(defn thaw-reader [x] (binding [*read-eval* false] (read-string x))) +(def roundtrip-reader (comp freeze-reader thaw-reader)) -(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "p"]) - #(nippy/freeze-to-bytes % :password [:cached "p"]))) -(def roundtrip-fast (comp #(nippy/thaw-from-bytes % :compressed? false) - #(nippy/freeze-to-bytes % :compress? false))) +(def roundtrip-defaults (comp thaw freeze)) +(def roundtrip-encrypted (comp #(thaw % {:password [:cached "p"]}) + #(freeze % {:password [:cached "p"]}))) +(def roundtrip-fast (comp #(thaw % {}) + #(freeze % {:compressor nil}))) (defn autobench [] (println "Benchmarking roundtrips") @@ -35,35 +35,33 @@ (println {:reader - {:freeze (bench (reader-freeze data)) - :thaw (let [frozen (reader-freeze data)] - (bench (reader-thaw frozen))) - :round (bench (reader-roundtrip data)) - :data-size (count (.getBytes ^String (reader-freeze data) "UTF-8"))}}) + {:freeze (bench (freeze-reader data)) + :thaw (let [frozen (freeze-reader data)] (bench (thaw-reader frozen))) + :round (bench (roundtrip-reader data)) + :data-size (count (.getBytes ^String (freeze-reader data) "UTF-8"))}}) (println {:defaults - {:freeze (bench (freeze-to-bytes data)) - :thaw (let [frozen (freeze-to-bytes data)] - (bench (thaw-from-bytes frozen))) - :round (bench (roundtrip-defaults data)) - :data-size (count (freeze-to-bytes data))}}) + {:freeze (bench (freeze data)) + :thaw (let [frozen (freeze data)] (bench (thaw frozen))) + :round (bench (roundtrip-defaults data)) + :data-size (count (freeze data))}}) (println {:encrypted - {:freeze (bench (freeze-to-bytes data :password [:cached "p"])) - :thaw (let [frozen (freeze-to-bytes data :password [:cached "p"])] - (bench (thaw-from-bytes frozen :password [:cached "p"]))) - :round (bench (roundtrip-encrypted data)) - :data-size (count (freeze-to-bytes data :password [:cached "p"]))}}) + {:freeze (bench (freeze data {:password [:cached "p"]})) + :thaw (let [frozen (freeze data {:password [:cached "p"]})] + (bench (thaw frozen {:password [:cached "p"]}))) + :round (bench (roundtrip-encrypted data)) + :data-size (count (freeze data {:password [:cached "p"]}))}}) (println {:fast - {:freeze (bench (freeze-to-bytes data :compress? false)) - :thaw (let [frozen (freeze-to-bytes data :compress? false)] - (bench (thaw-from-bytes frozen :compressed? false))) - :round (bench (roundtrip-fast data)) - :data-size (count (freeze-to-bytes data :compress? false))}}) + {:freeze (bench (freeze data {:compressor nil})) + :thaw (let [frozen (freeze data {:compressor nil})] + (bench (thaw frozen))) + :round (bench (roundtrip-fast data)) + :data-size (count (freeze data {:compressor nil}))}}) (println "Done! (Time for cake?)")) diff --git a/src/taoensso/nippy/encryption.clj b/src/taoensso/nippy/encryption.clj index a0b65ef..8224d45 100644 --- a/src/taoensso/nippy/encryption.clj +++ b/src/taoensso/nippy/encryption.clj @@ -31,7 +31,7 @@ (defn- sha512-key "SHA512-based key generator. Good JVM availability without extra dependencies (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds." - ^javax.crypto.spec.SecretKeySpec [salt-ba ^String pwd] + [salt-ba ^String pwd] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] (if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba)) n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] @@ -78,7 +78,8 @@ key (utils/memoized (when-not salt? (:key-cache this)) sha512-key salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] - (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE key iv) + (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE + ^javax.crypto.spec.SecretKeySpec key iv) (utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba)))) (decrypt [this typed-pwd ba] @@ -91,7 +92,8 @@ key (utils/memoized (when-not salt? (:key-cache this)) sha512-key salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] - (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE key iv) + (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE + ^javax.crypto.spec.SecretKeySpec key iv) (.doFinal aes128-cipher data-ba)))) (def default-aes128-encryptor diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 4fb2b09..a0d4e07 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -1,26 +1,51 @@ (ns taoensso.nippy.tests.main (:require [expectations :as test :refer :all] - [taoensso.nippy :as nippy] + [taoensso.nippy :as nippy :refer (freeze thaw)] [taoensso.nippy.benchmarks :as benchmarks])) ;; Remove stuff from stress-data that breaks roundtrip equality (def test-data (dissoc nippy/stress-data :bytes)) -(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"]) - #(nippy/freeze-to-bytes % :password [:cached "secret"]))) +(def roundtrip-defaults (comp thaw freeze)) +(def roundtrip-encrypted (comp #(thaw % {:password [:salted "p"]}) + #(freeze % {:password [:salted "p"]}))) +(def roundtrip-defaults-legacy (comp #(thaw % {:legacy-mode? true}) + #(freeze % {:legacy-mode? true}))) +(def roundtrip-encrypted-legacy (comp #(thaw % {:password [:salted "p"] + :legacy-mode? true}) + #(freeze % {:password [:salted "p"] + :legacy-mode? true}))) -(expect-focused test-data (roundtrip-defaults test-data)) ; TODO -(expect test-data (roundtrip-encrypted test-data)) -#_(expect ; Snappy lib compatibility ; TODO - (let [thaw #(nippy/thaw-from-bytes % :compressed? false) - ^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false) - ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) - ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] - (= (thaw raw-ba) - (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) - (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) - (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) - (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) +;;; Basic data integrity +(expect test-data (roundtrip-defaults test-data)) +(expect test-data (roundtrip-encrypted test-data)) +(expect test-data (roundtrip-defaults-legacy test-data)) +(expect test-data (roundtrip-encrypted-legacy test-data)) -(expect (benchmarks/autobench)) \ No newline at end of file +(expect ; Snappy lib compatibility (for legacy versions of Nippy) + (let [^bytes raw-ba (freeze test-data {:compressor nil}) + ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) + ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] + (= (thaw raw-ba) + (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) + (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) + (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) + (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) + +;;; API stuff + +;; Strict/auto mode - compression +(expect test-data (thaw (freeze test-data {:compressor nil}))) +(expect Exception (thaw (freeze test-data {:compressor nil}) {:strict? true})) + +;; Strict/auto mode - encryption +(expect test-data (thaw (freeze test-data) {:password [:salted "p"]})) +(expect Exception (thaw (freeze test-data) {:password [:salted "p"] :strict? true})) + +;; Encryption - passwords +(expect Exception (thaw (freeze test-data {:password "malformed"}))) +(expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) +(expect test-data (thaw (freeze test-data {:password [:salted "p"]}) + {:password [:salted "p"]})) + +(expect (benchmarks/autobench)) ; Also tests :cached passwords \ No newline at end of file From 4bc6dde12b875dcd00cb66bf1797b4abfd3f5139 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 13 Jun 2013 16:41:43 +0700 Subject: [PATCH 14/14] Update benchmarks (2.0.0-alpha1) --- benchmarks/chart.png | Bin 16818 -> 18627 bytes src/taoensso/nippy/benchmarks.clj | 8 +++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmarks/chart.png b/benchmarks/chart.png index 77608db95a14bffdd13ac1db76a3427c02c92d56..8fbbff7507c8ddbd4fb4f6d3ffca50f72b0f3abe 100644 GIT binary patch literal 18627 zcmd42byQtTw=cMH2ol`g-QC^YgA?2du+bnPxCVC!4#9%E2Mg}*?iSqMhI8)u-u?RB zdwcZg9;54z1+_}1)SSPnRdeo8Wko5(cew8W005$lw74n&0GR;*fIY#2gWd=crXB+T z(qS^uEi0GOiC4S-4svr3VM1~vLIQc5qE zyv2gYsQ2m%WPANF69fRT#?^!bnou7G0PrgZ)yIIK+8!_H;X(jE4?(rJ z#{b;^t?@tif1B)I{r_ovoBzL^?0+WyHrYSS{%f+om}OiJ`T^T~f03Cvh#P!eiLN0u zvba7O{wj=qQymH2sJ>PbiA}F}YLJ>>c+=tNgHtF`pAyrXWX@=9=^Is9#lsdDvv#^4 zqkiUO5Dg~Fl^!J>pY0dEPOfX;K5xLN4?s>`?x zK3Kbp=Wf8WJbU|BrY}K|&~l#r&2paFR0=XF$X7=2Y<#76ms$sukx24xmeAEa+TFYh z3d{nQ@2d&TyXv4Z76}(_VhIa)5ae`u1%9cmbjN;){U%x(%tffTfjg0xN{Q+}x$8{L zFnam0;^_`+`d)|bfg8@qSMU?r-c!xqAEle|<-2UiJr`cUwLQG-0^i~n(+!ojeUueU z@wZa80*of21W?DB2uYzG*IiqT#{S+qdts^o^gmx=cTmd8`tp0GI(P=uL!rEuaf6x} zpKc%Y8iLFwLF+ibqp+LXy8u)@9-!PpKe&71s7Y|-j%1{+uC7=K^6LN~5SWc}U2S9e z>!6mgg>$3a>)sN=@AHOSS^%k7{b_^Qt+^9LqsNa)XnxB$m!heaf@*d?&*2w3(zkNL zd-t8-Qbw+)iuNf}N`y3~XJ$vtl?+~+9-4qNl|X~|t(V=*pC+r(|B{T4HqYOCjz63y=^y@J4R zYN-hKRi!9!=-il?WCR-+#nX5QO@qeF?ruv!-crd#H33HF;!Su7Xh?&PH~!xs6VPR1Z~B6pX7WixK)n2cFz>N!H0(cqTVSt(6>6WM!fdk$jUZdW6Za}emC z|J=>pc%SC(j~~9NcSFiE2R}p%f-nH$y_!`=gt^=h()&P-??SxCGn?X*4}K-ETW%vD!@9QZ5}S>o!tus~suJP1%HhC3pi*!fI<%gY z*j%jIG{U=5GM^-m#V?_R9^f4#WI%LTr<Z;nthR!?z_1iHI^$Bx6Sx z80a8}bXC^CVB)S=%5)_tvekZo$?%Ndp7^;hK^8c5J03^7*5JrDW+E!G{Lf(gnEv<( zt)Ed$jEJxR9Z%O^+ghyWd#gIr<<*{4;Pzi+5z%Y*Lip0b(vw6pA%KE~0@NgLO#ikj zGNwbPyqP*8{-*MIMY`oj)ur%L;IEDTF+J$L=%A&XbI^+X<%!5w1?IkgQ2n<3s8WC2 zh#reF67FBo_?egsP;*_RU;Dqw0XrrubVzP`Gqm<+;3w}ETp5$cA9s<~jdS|Y-`zCa zVk8wFN&Zwj@ZT+YoQ!W5;;6Q@<>sM9!IG?!9!_t|$=omZ`DO-!NWBHnPO`qbu^s;!Hg&DO6x>?jx$zNN8-lt?$>atx0N4-TT?o8f z&wZ9UYt3YReFJEH-RXo0YcrUv0?}xMY?$2cGAh~Wp}Ph$+*hvByNuH9RuOe2T+^+^ z;zSl{YI4i&Y=OtXRQNwluX`cjw)5JqV$I7{hR(H;Y3{nsO8U36)p?+5k)G~_48C58 zBWp-KQbg9Z+=L_Vs_6&(UfN$4`;>!|)G;k$gJVx$UX=pqC;~#~{$z%g%4QlklZvYO+J#!=OrPI{htIo1yp;K zllP80!@g(yPO8#q65>dL`ZeyLW6RjbRB5TmDnh8!SJMI*aivBEWivc@P=(ZTM%~j6 zDI)!*T98&`+8IvG_y6E3V8xq}=le{OfSX`u&0WsaGvNNXp95QxARvI``L>W-hEMCn z1b@t9-R>91qYw8g$Zsj6J;xiaf8_YDv-1OUsM$v}c*~FWIc@L5HmTDYyQ_+Q!TwTW z)!=xeDN(Ea3{uS}%QkqQ%qwWC<_+vv`3e;+Dp>NMSAXpM#x|Iu3K2%FoSqX_ln$PtmUTcu zkN-1}z_T?)!1Y<^a$&3UA@lPk>pDlqn<)Tp|Ab|3FOxlquVHy|f|vpoLdH*uLijC| zf7LehfrqZet=}POi`7pH z3VzSj{^}iLBd<%C)L_eBPG9aW3i7bSkqQ0SVo_Wt)1CZ@HtB@~%CtiPlRs zM*M^3<+azKbmhxS+bc$Zmf(`Qe||}*>f$3yn}gv&uce~RWnwYrMz-P*#^9AFPhn`= zETd`cFc3Am8Y!N*O~b}`_pmwiGoJ;Hgq=bJ(q`>3{PMww<$h#(Vy(1JoP?6G5So4H_2>GGDo-?RedCi_H(05Zn5?YoE^kX?TnZ)9cAid4Dt7 zG6Zjq>iCF44X?B`tL)iI&lD7IR?gpA*R%lWfr(WuEvVx1&e&2~ml5oo9> zwj7pc2()L@^6Z2Hj=VZe1m&`JD)|5 zJ`f!DxzddUufVl(LA@!yqXK)EK?ly;7e(r>z*IU-0~2kJhEiJmS*(BGQiM9c9M`Jv z8h=rtxj?qhNYuYS%USW4!Q8mPx+63Cs-t4WxZ$(IbgGp22Q@P)jyG=t;dWcQ!4GdM zT)qb$A=v$P6LwzvDrBL^luR6#8pz5dSC@{Bm9~~b2B*3urwB9;(DC@q1oVy(KUVJ- zWnzR{F1*A!payVLbnej*LaT72Y2G}lbhw?4iY>@VPYYq?TI!bJ!1cA(UTzZ*P-3~EOk(`J^_fmU z4VnAd9zcePV?x}AA+GX$x`hd60|8n-Vr1*jTro(A1adP49mF40Jbw$el17?ve?=WZ zrz!LuTkW?-q~&L0j{4b8oj>pDI4NT*E@@+k3%9BJK1roTI8H>M{W63yc2LM2A4kfx zAWL=`j+G+EMBE1PydSXpw-pv_@kyjuo_AUYqN@#?^JE_3>rdOK+E>h$>7@GXBdb(y z7C?sOU6`@R7GuOamn>Lwl+4OrYzSud9^izw-LUVd`E={BjABT%Q!|7(A!=b=Gi>kt z%9e@#ouS?H7F z^i5lz*Y*D7&F{dZ=rJ$0KByB(@Y5B`gz4D^#~^6Lyl-z71vLD%vu@IHI4uG{(}K6E_BBr__KdQ)TR~vH zm&UwtuV7-*F$%`2oI0T?#RJN|lySn9yJ;nxW7aT+db~tof>?BBLMzMl_r%&<%EeWU zPspQ}Wt#A^m6UMIb*@uCf;iN2*ht82_KUwQ4@MY|2JhdgPHs9oVSgoba=+TMW7xv4 zG?eXZzJY;Jcs|_+a3ebj#5 ze{tf4JL7EAV4_Y)9&IJsXsN8h@^blb_9G{j%bbtVTU%Go%J0yCqjh#7F7I1V8ks4;M65b8cBMPD z7PDiq)Et2uoT$;=C#?JZF&O9HWo%A2rp&InMlgAqMOCcB>#)n?8>l*s?^#qo!A4OCebY0md} zyxgqrzpO3*87UZ+Z~)qpVpH8XH+>q=u2jRT$L3d;ZqGkzn_J+lSpi$as;Dx8N#~h6 z3`owfQV73;#4poV@WO%zwCfmPa$|Qa0QeAAzZR$9uVFn5;E&OgC$f1mkOFvaU0Sn*f&y0yJvXg|S@tWf9(SvRc9A%^aDSn}FK{FQL|`M*9RG~HaTk3> zGr&|+7NHM^Y}r=H(lg(kuMyoPwDlcRJ^wBc1md~oTKvvHVZA0SXC3CDo%e(rS#XLe zJP0NT$lhvS0T8z1mcD{E)P#@+SCV*`yE(AvRjms@^;O>qkC|{m7(IEo*bxw5PW?!T zvaans=T}}GTU;B9NpbF%r8O-6kY7`t6rmGe?P=Q~nU-;OgU^4*n)3M(od6AcFCN%Y zTw?e*%S5HXZMm!Bq|jHqU%Tq#gmgRSw%#ho1<&O~^t>C4wi$Sa|2L9EboyBfHm*KY zF_{R5C*grrU`SyzWpLmrjC`zxRW#fs>+48O0GcssN{e5_Ql_y4VB~(0hj;9mtb!P= z3QRNsL+V@SCZyi8+k{r0#kn8Se0bJ(ERXgdYm#Ci@K3cA(R-);K=VW*1U2?s+?4w&QDb@4!>tkyST!!>q;E(_ds8^%9TF{hVdtS~cl}_BhGygazu0&|x0zxPUI{ zTh7URS^BMQi0d<0^#Z<2t*`i2FIrZc3-mjU7&R71T#1Y zqpbXf&9;$7^#(o3poClW6=3^pkKOUVC$yO{TkeZ^&yQ4)10K6O4w^L%@OUegNUwV9lnc}Hm zM$=T?uR%>bItk=Oj_@4f4%sGSaj}}u|M}UE+#~*T_uK;W09hd}cLdsJMq!8QuhT{X z>xuDB`RTC&EQn_nNQ#lXdB~oRB3iy05gcDU<&o$#!l{1eW~QKZ?>ptkYUJI?6MP;c zbJQG%A>K%=>XFTFAx!)TFir9kC@q6v+y|bR3L4fwJm|SA2nl!}$JLJld|9(ix&kr@ zCRddV-FdG{ieC761>54?PgOrTACCl`mK`$yl-%6TBaTr-Hy?J=9L_dL9M-UX*L_)E z4f7kNj?qh*LKl!`8jh_4l(&n8TKgCQ{>+u!fHlW3uXy#9jTSvyKT5N`xQ0 zY|TF}H{!*q1|WmqYtsOnY$W2Z&=M222 zgs{(+&s{g1Cs?cV(@u77OzG=M>8ZrCD_jiw5NY=w%dXcb@4W(FOML+;HydZZPV5j> zwPF1gBthCu92d(^4=xYqhoiKZto_ENS^t->TvQ6!RpChbtD^0qiwi5@BbfpwUDw*# zT7bWiy=d0}znYA&c9`^HuSwx5|q!z~&P2gfNXDXiXq7D-N@^Zv8F@YGd&T)gUYf!S1dLLLqn$>3B- z_qe1#c^-AR&~1wz1%P@s)z(`rz=3q)-p;v{ZEl%^v>?$5fy+ z0>SgNU&p%Cq_R0$vWzSeo1(RTaOl7Q+@B1T5wQEM(SQNA<+NSzitMx7^(&9b?ZExX z_gW4Z^r6a>RV3?D;PNLoQpl&EN93l|LPYwzGt1N8&2tX3GVZei0h)9F+M! zZ$n<&ZrkhWANLtFyFVEU`3$lf?7hr2o2(U_d`9dRI$2Z8*pSuzr7ImdIrhEF35`q9 z*rK1Jwyx1i^XDvFsBackQOAK7-ycD!O&V^vF$qg0&c+>@bqx@U+?Y|6|y?)sI)A&C}-~N9b`ulGr`JdyC zZ*jzb;*@Ve&wmeMHno9n=Ei^ya3GBZPyeSN>EH8j4@=3-_B9K(H8jnCb}p%4RCFyI zn14_*ARLgTA z>m{r!h-6RDZQS?6RJ|ohc8oXZ-fwrGkBb@Ce#-Q!oH4=yC+DPMiy3tJD3+NWk3t1mmrBhd>{jLrhDzI z#Sc7Ls4bSo^_a14Yj~O8u>ZARY>gX3E+eikC0kjcMJ}%_<(p*Pbn+CrA(a0wI{#JZ|G$sE{|6Ht1KgxVK1~EL zwudF_?ouFPO-S!8&qYW-fo230{*6s5Tj_W=^x(^~0%q z@TxL~caJkH`|whi_j?W!?zSZn(lax0mb6RCmNN&#Y-VYD3<_63%fww_gB(>o`Mi66T+rFJ&3wECVVku-}Dxla3vfnY_fEStbrJIkt zYQ@@6+s)$@4>Y%ivjdiyZ#E8yEna zkT8l2PM=G=(Y5+w=diH^tyZ^|KU#HL{nw>v>aU6fco~Pu@kc#7qOMt~O~FA*=>lh;@L0$M7UPUq+m;-Tj4$uKtvf-gYVx z%X*x}#r%1sHo9qXZ}_DN{qlx$#z7!DYI3sv`C{BCTFnqNZm&sg_xUo|`Mkizack!* zQMq{;e>6EwfmKU3xASG^)U@VI%EZUiKinYe*=vGGD(JZtBDboX+kD#=ZvJumf4T8) z0YJIorjOg`Sk5=P`_<+^#_(^uVP+aQz`gwa?TyE+<6BtwHBIQ>`uuY+_^hkph87eM`s(|?G*QTPX-{)Hm{wXcFS_tuymkqZvEy%)!?QM%s9fbqNh z2MAV;!EZD0gs&!#xs7xU5&IUQ|4)keuY^?Wt%L?lE7foM*?Du|!$7`ePDQ?dMuxH7 zn~(lP4S3Uh`YafT1}L!fO7$Q9{{`e{{PKXam#<)JZ}-c)K){F?000{F_hIgz5r0cO z_`d@UJ_%^frjs`~iCq3NIcY)lu9?rKez&cte#7!j{tr(lPGr#k7tGk8{;PGa@Kg}=T7UL`+x7|jKu+Be+S=N>`7joL zsFj$dv}DlCqI!hHbYj?2HIv*{?b8_BE0n8K)@uv~@GC1TyY>AV_vv#2Z6RqU)&#jv znV$6La?rG(Q?5ThFK(pfkxi{GO1>&fs2k||-LL##TXyh&Y}uQ)p#HZ+*N1b^#@Mjm zsk{xNgffh-4mH_|1+HrE?Ln=zB({3N!iktmM z9WWP3t_gK*9}Iul$fp^00h*$yyfh=Wxnad_=mi6mIwf0l}KbDff+MhFN9 z^m*M5T))g2&!qfhGf(&Sh&F`Ep^A6dm*d`3zwygd27^}OhP5r5?de~W;?YYwA;rE! z<_!#Q#^dX0iozIKO36TAYKn@uPU%EQS9XP=Q0IGW;wrS9V;GwnEv0hfT7=nr$^LM%Dz`1xZZ-)y#m?FpZnT{8Ig}7z0VfmdV#5V$J$zAHO<)+7+9@r!Q^S z*kRq zOf=dOz3e?8ZJTsZmU}!br6dn-^s4U?5iMgQL5*+i2?eaP_ZG67NG4p34Y^)d15L6| z9{TceGl_DWxoT-3nYo$id?`!GY9iF49#{kxZspU@6T-#HKQemgJjO`GL^A9(6k zA@}_!e-u`Q2x0hLD{(|#*7kAEhHJaZF;r;ws4?(YDT4Z4m5Q%a zSP?AT>$J&zkR25(jCjk^il<0F$xorRzo)Sp_n~IDSpJcbp@=_%m){EX?;$HnFs)J# zoWm+Q!Z-A+b%(lRd5^>d90%W`@NH@r{Pdg*2mbj9yQ~>udB=3}=yh!zYqE)MMF-BfNJ= znbf<`PEO+$k1rr&sk`RZ)PqWBZjy)HFx!QM<{yi|HdSi!;SC;!$KK5;dPcw z>tW9~jgP3KNs4q`eEhmd*T%R_Es~;f7M{j+{dfsM9%g>_eKs6dyccYUQm(58N;xlL z@VEP~V7o-eLn@U;pU0>!Gx!=7v#JTeP9>zlrR*k=vM<{24r5--2KvDW9=3bg0<{QL zt6<)8WKG-Ubs>eW7pOKAlkAj1^x#$V1>KJA)Z5A2=^Oj@z9pS?s(0*!rb% z7WRN<%X&Qen6{@N*@fQW6;t)Zqo{LX*47;OaUHdT)`x%T&cMu{5XM5)Zg`&LGu(>Y znU5=@-tBN(?WQym__)9zzCW5H94n)6FI)p;O7%$k2fXhJI&sc8aRfX$G_{NHPLH(JW}l! zA%yGRi`37X>$Sjsn~Y6=;Kyx(B59AvUYxEn@E}YY`v!3?H^DY5~g z6R=)Euq$E#OOnre(h7G1+(EP}RUr;;?xW$rn&8cPkZ2|8b=YeC)c663!+d%ggTXJ! z{lHCnw*N;%2s`gO_pd(+QEt#Tz+V}FKZ(|17OCS?hzpN;YtjuMB9jO+Mi0)_p;<5^ z&jl&!*Ee3>6secv7~^~?lj-ngzEA4K^jFNu0Z>|56zhsD8Sc%%L*60#(Zd+*RXZ%` zWI9*nAFA(;*$JecsYlkMXd!}RO|{zIVTGn71Agg<%t`SCxW}(hw`9YNPB>enBn%$b z!p}`AS*6aN%$3F;2IJQ{WaCJ5TYeh+c-MCHQ`^<3T_Kxec9qF=@0ugLg7_UR3#a)9 zK8NHq9?f0xgi*0J^9Ep!sgX3#(5lpB)Ox4n0NQu&G9232SxP>{`X1DFB!3~E5b&dTiimk6b=NG3~r_i;> z0wlH<>S{3U=*Z|#N->U9D~c^lHp@*<#=8h#(4}U$pDVN~e&}bCjOwZ92qS`$j*K{M zf>eywQnrV3Lcn`KNk}RB&R@YeaM(Kiz%kSnp%RMt6OspuT~EhrZLyP0Q&WGmE|)#( zU2qUR#rQsFsQdQZECi>Zc)U}+skNPiPc*!T<7&TYRYy7$!4ZJc(pI)w=k z&$kYcf{Htm{_(Vo!J z2_OaoC!TdEBE`0LqlQS+|Arfj+s%5`P)P>3wg&&M!9Wbww>SLQDcW1T(>(mhyZHS(M+UH3*8UhCbQ+_=<6 zQ-{*gwZjJm>b<0r$7>pa!5~YRfX$D!PXo*u(k%LJ97X3Tj^-^X=g+pLrjKZ=I5^#n z^G0&!MtBR6gddyXEk(%B(eS(MFLxf{{O_-&PZjG)6p6`U;%n$ovrsyumHL&%{jrE4 zzZ<^y8QA>^B=aSDq*4;&mv2fX}bWjmqzS3~8?FWDh@WCg}fxN81; zV3xoPbyt^?oc=Q5J|1o@FR%D6cmC%xzI$46TMD?}=O?3xaPSe7xn+g3$ZP;jomAQ4 zZ~$ARWAxC;lRyusDS&2at>=h0{#RP|CI4jpr(N$ejIigsr#@noTk&~I4X2B zFZj_C{U=~F_9lq-_WQ_0avEljZnj7O5kVzqRtrSk@b6)8IDAvYn zN{7K4PFfp0P;rhwx*dqvGZ0+c#&pa$MYsI|Q~o9~Vk}OtBpJsPK^f`GclC6^R4?iF zelY)-Td5mW*aKdPs%j$kk_R9Bug{ScEw`2w_jr zD6P`tk$fZSg<#a51T_j=U0?@{&Gm%DzEkM@ql6=DoaL2~_QV%8!+Zi3vmFjQMnO5q}yq26L@SPsMB2fBH z4Bp+A6GkSDYUbw;hi_h7m0A#iGlpQ&|cFL5WoL5ZhB{&<|Vwr(bQ?TcdnZgl;k9WT9c_q%iK-AGGnv!7Jr2rC!-tF^pK z=t)pdW0_K@WtF_mG^G4Z2N&Uq%6J*#%3okERM`-QtnvE##>T;SM)`;E^?>(&DCz50 zj@V3#E`&z-wwI*x^F16Bp`T;33pcyCr-vv_Q3}jWtR=F1h~KNV-Ed&jQt_~*qL|jH zYYFfsVI&dDwBTYSi;3}1wDAqqVNUo!Z?l|_{_#`46QieL43yE$E$+eD#mREUG7g+#Fj+U6^XbyNd0|$V?0gnnCTsfI zrBJMcg$&9Q>~9LkNyvfS#y->`Rd<}JLyFz|#A|U0K=4@$Nb1U?neYZ~;!}=;euyE_ ziC+71FdHDJ^I-xwyqZ)Q@L~7mkx^yFL)^N1##b)DVZAl*rf$)maL4W?68=U;4HaP+ zZX&~^M@Ewk2{V-$zHoO&?5@-m=9?W@UmJu}qc1faT)dP@m(yy$kS|VQx!qjE6^p=o z{q4qXcIbG`@H@lRI>hvRY&L%f8Rwpbczx|$}JnYlQ6v0m^fIW87ROO(62bSZ}Edy z*awWq_&+*r`rlDV^kuTj$ZaR9Gc5u|j`_ZD-g+NB4usL2a4d7&B;7Z0or`vBYfYB1 zg)GL%lC;%^PvRYZNX>Cmbz4u0<_ME)kv+zBI#kQ`R0-C#nyUf2esZqiDyQW18fMAj z3#u@+L6%&TSc3w15CBwZisa#GIcYp;0+V>ud5bk_gImh0r)DJ1b+p>PmF<-xt^#CV zoVdJlqdgxEt?SCY5P;hm>0q>577ZDX1P~(FU)&%89dQ}FQzFL2|L7c zf87ovo%U0sHCvm7<~q){MTUTeUa^c`6zU4T0%U%sdkQUu%2 zIg$5QrV&#mBUZ?u{zhiFZapm(Hr8ypeshD<_I)QZ8qblaE7fWtbSf$bI+PImAB+^_ zAN8D0E0SiXWmhVf>)U(YTaS1@SWD^%xA)4fmP9t&^V=47BYoF+>!Mj4)1qg-;6)2L zdd*$WRtGSw=tJBvgYF8qUT+uw^Mkh9x*~Eo0|Jz&?Wc!?Qs|s@Q`qbDp|>~nxka$8 z-Y1i6!)~C_kQj8;sGO7(;!be+tj;Hi|6E(9Q5dr6rs4f~gmgRFhQ9!S{VIaqW@Hbp zN6^d+2%iGN;)gLPO8_%`LwOOv!^yV1PZ&MEKELL13+M>e9`WpZ9jT-q)h7^{Fs8Yb za-2YP^o$UEl!ApY2Aq80hOEos5jI?I*ayo=MH`}UY-j+gaN{G@GcqCthuauS&*@hR zbhN#&4Sst>J8G#vdjS0e762fAUv7olwsR@@uFJ04W&rV1Sv?HrH7k;6(;V)3T+kG5 z>s)ZWD`_mVdo_$;d%K`^kS@6tiOExGMq{(PMHK%7O#ac41w8Ub#@7D_>M5Iq24(|IuiKRpU%WHkOz z2@NL=E0RFLKreAGXfT>c)l`}Op_zXm(37gKrLK>Utt_4&_CrfTeJL`m*`6vcpV+`q zz}e^i^X}*C^iQgq;nF7y@Dt+ZdCS@8G7<(mSPHWC;Y)FruP&s&LSd~rAZ7So zar<3w6lwu%w91l3DC*euIes?D=UTO++pJ?{MnMKe(Hgv5x3nkNSzhP_Y` z_U`aa5^;SEqfyPnvFyNN8xx)|0f;aY>cK#NGV`R}g$vIgi1?%;E`yJsDo09sP#J^T zdNO@oa%U*GgD^+5mRG)fRAarq4H)-qmxBG&JMdRJ%VB+N(7p3qu(dAkq3@L~J2ry7 zT=%ZJ8+w;bYEsSbE5o>wfy)!D+!f}!4=YN9Z1?Yz7-3!tco;4(Pl=#h2qffu#}$Cx zNc6Lf1tP7(td88wP1M3PJ)tO~&%AGWJK0MPk#Ol;fK5L@x$o&_?x+j(lVjV~}H8%m;aluua;4&p2j<+UYMB zG1}mf6qd!de6(|xddmZci3Vnd_9*u4lWT^`mXC+Q`^Wyq>2~K{XR7%cqc$b7iYpUd zRAqItecs2;%jK{}BOo9H{Khx9+(4>1Qh)K#Fbh_+isHOIxh=;#Tsptpy#G_UFmfqW zpo=%)C<8csb=mkeaXsoquYFG?8rO@dd7{kLLs1{C=exns+eS0Uv+fmqTrgM&74*li zohq{$zpTJu00%`09SfE1W+gpFuyE0xVah{Z1YE!8DC87V;sL z{;JI2Bvx;=AUk$cIR zhFW}3p3knpcjp0tn($lls{Dd4XdA;fk(WEP2|drCc{m>=Otd1HNBZKN45IiJ>iQ?j z^RKwjzv5c|Uj)qkh90ML!9dhZmy{e3>XV%2%T4N-bTj2bZG{*PuO*UL+IIyCJ@tv) zR*AS~4;>fM9hB4-P3Nhu$9*Mo1PN`wKC=b^J7sGiR6yP`%jiWXRP%B}@jT_Qdiuf( zFqqF5BM2}iM(DsAqS0H{`_ux6D*n+Z%Zz9p_G3h$Q|t)to7yIn^7ng4ZhQ?Oklh3> z43f2c=q+hb2bq@sC4r#gojvo<(&>OEtdz}F%_Zd*2 z=o19sG3+;)f%VpA+%dRQ;ln7)reis!Azj~RtuG7a>cnhIA<@+=3gvdyH zAIM=Z3iKGpT{YK>->nzB0D++xc;M0FTL`|*~vzglm5x(U%h?2IaAA4@RC3l3Rv@fHB>9u z#^zg^O$+T?4zeOX#)pU0=hZpmUaOS|?h=0~Z2ckRkuG+^p0dGrlwZ1{S&)h9hr|4g@c~E3{<4DUWX_@3&8USMIkpz~XS|{c$aMm2-GPcz9FU!au8D z)_oXK`0Q&^@bLrlZsX(v;`RB}xP4s?SKDjm3!p;}1_1b$Rjrt1{H5@T<~LJ}c4k2m z_k_#G>$C-?yW>AQXwkq#PdVIzJe9TX{`k@PWEjS+;TFda3$d}cjQlS0Pt{l~HX4pI zA4ZD=)VakBhk=>&1^94zLY*34cmA{<@~*j%-4*DO6}wQP9&ko_vRCshOIs<^{&HGv z8G|+}^;F%Km|d<8I&{ux)(&<(F*9B?mp&a>@_J!;e44cXSwHN05=n#r;`Qu7h|dCc zkrO>&eX@g}l?+Cn7`<5q3JR$h5~4R# zt^L=Av;N;$t>Jmt9GHdm{l4arWw3KMg!%J4g; zL|$H^jB;o3%I*9y9=lFb5N{L@QUrMm>Jf_=)?XS!K`COQt);P1CzEevw^{}sQe33p z*3ow~isPVn+|nCMOqBoF6(){{D(_u>2$8n7`zghakrYI)T%g)d?HFtsRn*P0$pGV) zI|{^nyLt}4@oY47aaChg-2A~67J@&ZHDbAQzz-9TDe(66bag$bxnT#aeke1B$o`sl zaxesrb*_r90);nR5nwi(+O;CL(Yo1>ZaGS`kIBZu?jt=EeB;n_spdIH{m208PYj!R zhzu!WInYjGOFY0A8oD#tGi}X!bQR@Qa#kM-_bD9St&`A6w1Hw-B5Klm==F^!5Y@>BT78|MqU(?Z?J%|K! zuuqkCrOxm3)gi5A4#7u3=KNs2OB7H9x+|4W{{UZ$R4l0eWU+VI-MwvZ^0K!lM{Xmj zZ)QIcG;vq~g`|6{1;)SvEz#JJQ^%ExmWfK=+E?RZjw(8Y_fGfEaH3fX&2T~MtxDDV z^avjH?>2ZZmi2tz3nUUHZFk*%WmW^mgEcz~Sp*=F`v$@nA|7l_t*;yt(K~3Hq;=$1IHFvrRl3Q^C#neB&^chW z_&p?W*4Wz&lLj_XHcw2g2Ao-uak_c%MjbnICRoo6DXzWi>?{@0Z%dL@$AhXh7p(hX z!@F7u1V$Sjrb};$md%}(AOkV9YD$J?6O`p{mtiEmI9@V13s*F7RSpeNA0TFM>a01d zt}VRWLKc~k#ldP`ztWSc48E6A?TI+~Gm}+k5)av?%(GS-FXF1_g6r1B_p06#E)SGj zm|oPzPCjD-+S0(0C(%*l!rK!+mfMBrH;#5G&6vT`#85eSu9c4XKZ!LwVVi{gugzDP zDZlOKww)_m{lvL~{?0j^VZP0`Xtl5J(O-+7&ENn1t6FjoullmRsfM>dHm%*C!*HUM z;Yj5^2FuvRwv!&#?5YmFxcBV#fUUBN>dt)J)BWu~1F+X;(3>jZwWVEXYfQV*-Rdbz zcYjIVqI~7wrPRa685lsjdmX@QZ%~N|pv6Q8;s;aY_Jfc99GUwPD9qsL>gTe~DWM4f DPYMG! literal 16818 zcmch;1ymhNvnagL5L|;paCevB?m>gQySs%1*Wm8%65QPh?(XhxZ^QZSJ?}f~`G42n zYcZ?$FxBN%-BmL|ax$WD&{)s_002&0Oh_I80Ph9>K=z?Pfp<2(Id1>}{I23c{E9AG zN3#T0Klm{E#__CRKdb=?uF2tj)T*I-K-l?4DB&y2P>s;c6w#oCxia?;NSP@H+)o}~ z$nIJ!+rWWKz0#0^004;5z-gd3@bdSpn;7_04;gr2_y_>_B)mQSpEEuwARV9o6ZG%d z`yzjX{yqD5(D&v3vB=+`f6xBE;(x>X4<`P``uFS~T>le)BA%eq4^2;Z*>b}$!N(R< zCEn4+^+`;i-~tC6tOb>|n#yPeXF?yD;o?0$InHX|ZhztpPalN@0QjUvrSI&5hP?7v z)0H&pEQ1=Jf}LKpdQ-d&+bjZP^WUX7yNf7b(s%WcvyYT6Dbq%)V#NC_b(`0p+JM9R zj09wWXTHw+59=sB?3%1#wsk*2MP!EDfL~b%^AucMlqsLM|JbtVRGz1oB^9?s7D4p~Cou z-p6Sc$c^d^2;XGpxH%Z@NAl5t@YEs7%KK=x+*m@N{B1^7)k5nU`;d^30(`=^VPuG_P z_f;>^JNyO%1^$^1orkqD?_u{0jybYa=@z2mVXF*IelhFx%jsnxej^kew;3SYjTA6!1_W+ z;i3wuJA$`9h|dPTw~=oitLm>mJMCdVwT**ceiwM0)=fcwW)qLx;uLK{{0kcvWr|ih zToN8N9hWeRBi`|Y*xq#;X)Ff`Lw-)*>1gfZ6?}}Xd{>JC$FxaVok zKzktwM2(FrWz6=(I5Rt)8P-_A)ErYH8mwkq*GXwpalgv*rb>dCT%_Q&tg&Tow6~wO zu)A;HOs#nW?#`+^@Htln1+zx}G8ASw#6SEvNAQG5o{`#+M+J39LfX~>5RLNsAWW-? z%#ekbi`z$=p1@LSBmuQ4P&y&sOq2zfp>p;&1nIKHi{=!sTI-SemCNuhail0Wy|z%tQCT{O zWuq^l4-wthIXOiT0Gk`pryEBY|HWCy&CJ}e1z{zq2r4T2w%O!iKHSz;yMAQs&3}m6NxmNemm?E&<+3j8m~{`f zR~r{E`Y*d*F|bqY{gfieRJ{!9Q_DSm8bSV3>Hy!ei(wxYP81AOou1aD@P#i6a7Wwg z_XyX!D&>AQL-4LEN>i7hR{>p^P?;+L1PP9W5vyXYmX1dM=+&*vC!aD!?$1C(*xaN+ z?~?9zQgA1BL~7SyP8h_tnp4h<5qJ85*3Q+68V%R0jo>jH`~H~NByXK~zwdckCCbO& zY#7=}a7mRSEjX~F!3#aEcN^!C%NdAL0oer!l4+++4|ldKGZ$qiE>qS_3$JaP4C3!h z5rYGYF}eQW;T}pjNELMrH2KOk@d@kFruK>mObQ^nm=cg5`9K$=1!q}=K4Ss8o~?g4 zr+snvp9Q?DBx-k<&`EFIjA}2;)%e$6X0oS!bC>{(p#C6#o{~?PuPi`m#omHB?OUpf zd`ndZZ@DS)EiJuW_}-Gyzu)&UK7~(RWZk3>=Uu!eWsX`f5kM}A3XP$Xi$x8bYsr(Ia=$^|jow(6Z@4>2t2;TC&E23( zMk>ZlJitgR=EBE>FgMF@Q;Q*ZVeoPnicbzr^H5EYE0KH|yH!o>OWeX$hcvt>1AAK~ z`wQ%^0%RfaEy)WJTgNiufr>*h#w%-0aFjx$*_`>kXEpZ`W#`M=E?UY@yONf<^I3C_ z0|<#*x0-^tP=;?}1q@z1JmUB5u3t%{>YnYyI$X1AnegjJnA#n;Eg3(do4zx%nY-2Uvc2W5`A7ug4P9-^ZOv<4q{pr)+r=IpnrN0k(#R9= zMcWihG(NdP_rRRu&9(tP8=j2ON73jlH&Hj?BW&vtL$;9!N#O(oSBj+xn&|M`{{<;mgN*?99W@C1;W{ZCq zuK##m?oW~}zkvZFc|A)OZB{&|S~`Y^-);cBE4BNWfqOyNcJ?!CSCci3iAIw&;C&tW zH&^&~&-{1y{M%#a^iSah7W*%leO}$?_i&X_yJIlu6b3Sj=w>p%|Z1N@(nbMoG zc1KNS?2^pOutH+5Yf^QM?LNPwZoe^L+qu|pRxP)bQy>|qB(bie8ekn@X{yOUM$=D( zhAYH)_^iNWT5%#}cV{q=3L15jgpLBr3$*oV+DO%;qA{{hi(c`h<}rf@cHeyrP@r7wz8Fx4 zvH*S@kq)TGaUP9`*r|U5OqF zxf+2oLEPlKFMUVd0@Wd5UwNy)ZjL1U87=ymG@oqEY@+?0$ZdIS(I>4Wl72}r^@H}f zPS)$qf|}KsT473CmSkueM9;tMT%2Zjr={x1+Hz_nZ|GQmB0)kitVhCg`kU*$M66h9 zq-xVbFN2O+o2IS zwAzUv?7yy{3c8imCR2py#^ST*k~Om7nVSb85DF8zu9%Y{8{`i^c27XK6@8Up<_hFR z17G+sxgmLQSsq}PYpDI_GxBrNMVHEze8zvMHQ0H84rO}x%D^w_sI@z5ff`NI$&nyx z)F7^_@gOPayuU5R396f;IfU>OIZ>JP+J-R^YK#gcze*^J_aJUhm96Qfxc8cMH7BcH z2I5dC&1He)sP4I*|ILUv{)2LMfsY8*MNtl!{L8K+hBGWk9yFdkG*dVf)Y z=CG(4+8Nv}b5o2e5}`y6$^|ZOeD21xm1V_x<8AF^`DL=d;L)YUbKcuW*SF)4=sHB36y{)RWWw`VnbJD2FXHUcmm&_7KRvR!$0Ef-h!RiEk?aqr>0BZ0+>acM-&)V@0>G8am z%W21q;6fH{M1kS+&QpUq8t27KK-&1UdV_lWs71umgMr7E&v*oIO9&i3^Ck&Gp&vst z6FkyHwR<)Z@KB78!JFfVIqgn*y<2(BGOZSGH%F4rwtlZ_vddgmI&JLc(;2=RZfC#V z^}AIBYQ1ma06MgXHyQl6Sp^F$tIcK7VxFnyy3?^mlN-mscnIW@6UUE1{BCmnPxCx) zK)OCKzzmJ^MLJtVr@q**??qz#R-9<$GFYL>KUSPM#=jb+4%TbY%C{*yQYiRcBzyMls1E=n zG$T!aY5qioRoG4u->Q+RWi&rmdPXSTl>;Ag6r(g+I7s-r|H)k@t`D45=Z-HohiAEj zxQF6ZkG+|mL}6M?y6H+F`!IMhfqLjbK_&59+O{tRMlZ|ukg1)idamK8w)n=4`vlEJz#*BZu(b% zx34uuIGrUY4Hg<*%-uTK$sJ=WCc9QZ+LUPBIHQ~jqsa6~YuxIL7AcY#EtsW6Of0Fc z>3U?Hy-wb>`jQj9OBWWQR$DN1(`D_%8{`vy;81yR&)pZiKT_!PpGqc!%fXaotr>`y%s?wm<3AZzMH8-z;7k zcNPRN?nhIo50@u5da@Oj=7tF2Nyn06DG1GrRps^NpmQwBbtP^o-*;04sKEt{6*)B# zf0VKz4iR?C6QS6LHwx-5Hb=@ecB|$mP9SyEJ&Ih~X>76Er?f5+ZF@Z!DtEfu4B>W3 zcqF`xjwV}mw|97ohS{8Aa3ftC)f_(8JDa=1828IohPmeSnzF0d6MYVWFRy6rlKu2X zW8g>DtOI}3TBt>l_mEm7rpTsDIC&Bzq^tjDzAJiCiO})fsbBX{(lDYe zz<7|?>6=!&{`2l38~1U$fcon2a34NChwJ@A3ZvO_!|U{Ku6ZJL{kEnvS{BQd&|N1s zBSHkK-Pwvj>)JHZub)mtpRI0IlLY99~mM5+$k?7(cKZci?A3Xj$X_jUj z@B!P(YKk>Mx{3lm%zK}}E29D@4c<`SH?+I}#uGhB{VeZ5Xw*bLJ;Uq5f_Upo4c%I0 zI1dYB5qY00{R4`jL-$~|?J-SOUopC|9^2{iBO~i&2$#AL*{EqB4L6tl1(fCTeZH*s z^K}ay?_+>{tP%2B7Xs>Ey9TZHN#y};3SZoi2RRDupFrGnBd``3*) zj%^w~U9z|Z&*QXGjT<><8{MRGR@2wXlU;26@}m@)x~t>hbauQR?fKB%a=qT8-NNhEzQ^)cP!8q%&)nRZEkck^ z^6U9htIjXY?qkrD14~@V9tw0ezdy!`t73GOZ{(@*$ABsLVKv&X#J^IX`07MBj^Pc0 zrgG(_GJWR`<)zemUs%#lm+L%sJFh)--0h76sN;}|V;e(VXf>?) zza-H7Xf5gV39T(ko^AKmaqNGFlDIRYN-0$rev3;Ia%8}o(UhOn%AU6%&Y}z=G^|!V z=wYt5HeF6I9YxMo*$*R=O?#qqcZOZuYB{yN>qB!r)ziey2iw=7JL&u$zz(B2;j8!^ zmJJiSOII8b0+Npq7OqP?z$QShkb1k85OxIf7rcS5gyV;*cAIEeBi)8m0so}TT8C}; z+mT3fEUcfCUE`&O9tStBte3ddIXdU#ysxw4?q}PXdx@7#R@|EepSSJ+O^X|{K> z$JyaZdF##EdhxUOTg8J9olyR5Z+G=9PV|aw{kG+JFWmpns{ilpg8$>{J8-%G$L_;_ zP4)kOi~qe{@Q)V4e>N`u@4F8;E7+gEyvSTh><=J}!vUF(76%Hn+d$d7!sWO-4S}x2 zVhX|PckIt-_sN0%rie6W?}N9TtXJjudNIE^oe%c2ax!}J-l&n;`KCIocjd-n`{HQg zVYIz+U~=<5^>{j&DSdjSbISOAS1^1%EveISYY~+ zs~&Fq8tzF$b@@LZOHX8NuAq`8R-`DGd@{l{>Ag5z7^yM*Q}2V*B*%@ruN7FoYS#C{PIQWE{VQ?1kpos&6K zoz35SbEa0ToSXZAY6RySe|v>fVWG|YkU*23Ru#GfUZXI#`20ZP2lzLOpq75>don3@ z0v=tAE-wapwyn$9lEVV$2CD9U(n_Z;E-VD|R`3d+T3A}b-QFB;q=Nu_*4Nkf0D6qK zTCU7`4hA(>=L71=(_~}iq45zS+#|ccTr?TdQ5(P9^S>%h;Cbg++(WtJ0s!?)iPPrk zbksb+2!oJo$3S>6LS1DdIobO%Wx`4b{j($IQGpj2gb}0Z@XAY7&{fzrVrL%Q8lBY= zsKQVQS=-#sC|{};?0#d~5~qih$vy)s4ehA|8m?K$=E>9dFbqsgapfxwzgux_k2YKw zg2=~0;fBl0pYPR{*cIH);?+%s3~wYD2Eh}BDzRT38niDWGiEgl z3Rd*-4&DcV{d4Cm6$x3dKsp5Bv6R)YRlGN50>$6_2e19@Z26z&{Ex=n|Fsjp1N>u= zzg_3=*}sR5|0(`8ncr?C&L~w`lZVhvl8D|GFM0J5HODBF{pw z*8Gzm`^#JQCVM+z{dZl*suz>8z(0gugv9NNMO}N2yKhbry8_`YQGohl5H&ZlV#~rn_jLj zGP+=$-X0SWRL{=jYWTx;;GC(Ut(^IzIFlAuk(5k0I^o1MmvO#|8dBIU zoz_X^>2%0LgsQPZGtgfFIN^BaBz=99T%uzd>6D_2xz!=~X{A$2po(=2LTg$BqgHCUkt*t-C{fyu zy?Aq3f-Gn9!|Mmn;`rD8$+s|ZeRKR$Txb0O0639yQc*51_3(1ePJ3>lS(vD)41=98wBxp)YUkpw4d7qW7RBqvs?Q^39x_49%v>bJn)yr6k!nf?l=dZYc z!v0KZza6BH#C3Q%8A64Py-F*Su4OKwXhaq)?w*W^9k{cQUhetgvbh-X>dETapg|$x zz z8o{44xCNEAmIMiKipXyDMZJ8rIY$BGnx*@m|}AAFz~o^rp1GP5I=_6YsfBMar4!yztlFdp2)j<=r{kQ-PUH zW$5AUSG#x9BS0H3IE|wX+ZzffNFdyQ@jk(Zy%ECauMh`JtN-x1caph%fJ|`HmBzOZ zyj8&dmr-xcofO39>W*uiDwm8`C?4<^Z~q>*{^D{{1H@-gRDGN(SsV7lUjYiZzK`|R z<1#3~=4}J@cc@x>D>2t^z{|YVYyJ^kT>uDt+u5aiu_JW;n`}^k|BlFav~Mz73+DQV zkpFitZKzrru5SgUr=+w#?zukczpK@lD&9DL#u+kPIt_2^)S`SP<-TMlkPWC3KB-!> zy7&nU4E2YdNy*)}4MTi{OTA4Snl%v8C&3~&Vlr@IO3~ID!_7O75pjfUKiTa4C6dIy z{QFVxU+Mly^Bq<1B|6f0uxGi}l=tw)G4R&fpOX`-bY9Qhh3(w@O*WPC2-wz#=q7a9 z^1SG}8QNI;b8|(?*gas1(y_N^^2%q&@hp?$!8&J7m)vmd@!0qPi~zvw%GRGnBaQti zm^~jzjtaT@S7sT|FwH@%yr1?{VM&~hD;^0Xy4Pr?0U~|Ef4q)ZWQFaq3%X?tGK0q})gsjikv8q?hvI`Rl(x=Kzkb_gbuv?t5FVo&IB>{l$y?=Z#n6Q4R!+%2k6487D$E zF}Tmms3{PiS0?$TLH%@*Cs0G|Rr8b$-6LIL_T|K;qtTCxsY|jqIm=3iJkCW*%@4KA zodl(m-I%ehAHR93`*Wt=*^ZXjDK*PW#HzjMu@rZf7l*!XSUeXK%`SvLXP=asDiOV? zJ+Lw_=h=IuI5Y-JC&Y&#@t)`;5vs#hUHA4wG%1m2ai4qUwOJbs+DPiIPFDyX)2gal zd|RtCJAImxT@Rh{mZBc-y0MvsnLax5?Pgs!TOLs56=j3N-*;>XrqN*DS?e2UrO<~d zZVao-)r=chhCNN@k>J6vRKspK`jL7}{dpzmwvZn;+Vuq-(yL0vzxkseKTg~58AKzU zTx+{rMgg$AMXxfcl&L&jl(nK*T%>%D3a@w%5WJ>Xk}ECRR9tR=t749%|27+5nbWov zbZgfzwEVd1&1NIbj0oaW0uQ5z`IWl9(BvG_jF`-4@MCr0J(O?$Zzr~y;{=qn?-p~z z`cB%Q4C%0(5!P3jZKdMi#u>1m3BIoyR43g?d71IdyF~>X`XcJ3#t+Qgd?$Eti$Qww z-kD+N*-LZMZg|zb|M@VynV%MPMlxXGVSP_=Ww}#iH;AhVvq?lKX%u`y{gt&=m)-jX zN}XREs+DX{;Fjf9eVBKG177=ElVi<|Yt?Kdk(g)K@8zaN%Q(txu{@$>XqiSy$xBea z)vzkbL`pRxYx}!qu9{sAxwHHo!SQ)$)=R9OH$2V7R~0FR`Z!ugm*YPWEk4w!|$C2Hu5t z?b$U$#K7wdD860)xRjUh3N+eQe%#Ubx@6ctc1_~yb&PB7h%wZh@Tn06jGwDSD0@)6 zyK6i4e9lL)5*=*9bXP{(R~iDzwKp9xjIKbjRZxF6!lzO)m8Hnj!@u?dHZfgVt<+8} z$%0F}zviZbo{2DFhLKaF5%FAP{I1k5Bzne768A{0ls2s4pxCyibu2f~>LfrDQ`m-; z8-zJ;dliM(b-;ndieyhTHtjGxB||E0pNfIOh1@7mN7 zJUjx4ep`gla6n+vg=?kpRwuOyG@x0&NLuxwUq=N$ zDGODj`GLII*O0(a=ecb?3!Kgu#y(Nm*w~+?;)RLtw-{x0QDW!|mi@Ou11*tE?1?0Q zO02aQ!2y^1h`r8NK>C_n?l49LY=!b{-j#+?G31zp{Jg~_sNRK_!yQJHX8D~SKPim# z=CR9dVa|vsZ0NBsnzrZ7PwlGD^+kCVb#<$}GLv3LmgmA0bv!3Wr9Bm6_8lOnADUNT zrc}+QN+a)Cpgj5sM49d_wZ4m#-(wwS#}YFY%EOE^q&^OR{1i5Y^{p2<&GfUwP`13U zOFBzWT!$1Td9%+QbYHIex0PW?o%%x>7fd+I6!|$=dfShjxefDWw>P&ksOmvpd&e&3 zCQKAbbt*@GiE{1{EKlFQ1#dro94+bYNi~%iM?6r}lebecK+53H5f+k`R>ENUgb-JG zrL-euoe%7u7)daId(VJIv^(m}8Og+2K$qCWWa3c%k)REbs2?%4gba^n05QolY{s=t z{d^SFYd6ZzzW{zSvC=sqQglQWL9~%!=rqiZuJgJSfsPVZHg7m7rKY4NT6Uz(c*@tw zb%YV2ikM$>fWjAx%|`8kPmB4Zqj0a>|bjpb<3zj&H*h*sq2nHtA?z9R)$hj zB(>xnKw32tZ5uldNym+IC(c*-S;_*d4cA(l#FD4e5|LYcV&kx_=Ah+bo!io6qmI|> znO^#_vX99CPFLd_zSGT&!Q5`cn$IoR3O>N>73sh#a$W`MDf&$du27&T!UR7zW3=H2 zp|lbbnbl1`H=bbq28)ye6DEmQTAxT|3}lA;6l~a~_d;faq(PchgJ0Zy6j_Fq zCos))+DXr%gl81yaO=1>2U$bbD;yJYP5xu80N*leGL{em9OWji<|Jdm_g1AS5GBX| zT)Fw0g#2WqXeH2LJH@_PYY(F67v0Du{Q1LV>iG;y1ZA`OVchd)!k_%>7mdw#507>NZ6{@JqDMe}HD$e~;DCU(PA^xGkDZ%v;hcs&uHfu43P77;4KY;Ge zQubesv#z|w6?JeGr+&$?O#{7DxyGL;le)_!zi6#YI!|*7k+{5Wb1Lf~yITGKiaI0H)31!07<7K&NHl$2jH+BM7uDd{VW+(!h_Ac852 zs`4~1gDK1us&xWEg%yjwODHPi4s)5W*q*L1{qSETFr`gRZQ;5Pg66pxxlKLYdQHDF zxP#!Osy>oG;|2x* zhr-3?JI#_1LM~LqFXB@Z0N{|VC0V}f-b@A zMEtP^0O!kGWj25cJ&}WmGf%>hPwHYkt8l;s%(8;i6bs{kd@Z`Yy)}BpX?HAb&Is(f zuN%q4FIu09=m=yz=bk-A5_vwB?g2!?hQUC`e+@JdL9&@45EZzf&h}MP>Sp2gaPYA^ z6$m-G9BPFJxlK*H3w%&tU)zFnLc1`53#St&4lds_S}Jn#pK<7^$vb`YOZrZ;l~V+! zP0@FoX|;0b!^EEKPk~z)FYpae$+6f@3Z*J9#_GmNpX>q)s19JN{7Hi2E8p_0k8nGt z1Yq&R&_j3$+h~v2u)%te{lUHkql&c}RLVztz^4;JlM=_B+AlHjL|$~}M_@TU{Rf0V zBt;$&9}o8tbf5w~BzF0;cO^ZEuGGht5JD7lNt-#9^Y0of_O4@}rven&2Zu z*F}j$ab=E4kvfevXWXm9*j@(@NkBndiDI%ItzbLS`s-&3SmqG=38`B^6tBgjp`t*N z;^Z<+k<$dlKoW!@Wpe~T&IJ99f20kONGsg@LJR_IPv*PvZ4p)xnMocKa@pbn@E` zegl<>jaJ;iTY<4MUcfmiiQAJIB;<7Hps>n2`+P5CqvM3S3?VDj6Io&qmLmKlAa&;R zw{UmKOuB=6yw$aCuct-rw^9e6(827fV5vYcqH#L9~C&+Lm?YWYGb zL77=|K-WN@$jBe&pY&sPKUBa5P0@T@cLe}D!EzFvQ-rvWQG$HT=K{-Blj)2{YnNZy z5X%uv!EFt$g{G=IKUsM<178DwT0)ySueu$W66Q>?UsoMkna*fZr=^!nxgA6f@}|L+ ziBH}$z02oAu@=llOyFv|%}WQ(D3LHFu+{SkP&NnbjRb!W6++3}nRfZpq(8U8r2rtS zYQAO3nsU@()v7dk^hCal1vGrKo~mO$81$Y6#^hCatlz)##D8!oh;9x1xg&5%h~$Rp za7sow7J@L!lO_PDt4Zo$Pxy{Z=tkL8#Hto5Hcp;xlh{IoTg9XVZ;9JjXPK%QOBm*U zD|S9PqT}@G(?Lgw1361Qg(uW3ap ztCSg`vS~|4$}{TlkuUDHJHRBL^Gs)fTJ2DF6W~IH)ZZhK-P(1xN}%2U=Z=3bBsyPx z07;@OMG>&1$|d+w)VTDPCPX*l2Se^vnzH)$F1%xihUVRiGy>P)186O6UwgZku@yAJ zA-JwTX}JMsAMHL?YqV37{UPb>>>U0fz)6d&{^`@F+Tr1nXLXX9hey%q#8jt7CWcTn z;W4!@r2~`Y=Fhi8rBL6@Ed!~U2MM*TpoEBuyD(f#_z)|?_3KA|bZz2d&Q^Y3mzD)l zA5T)iIk?RG^?IiFK#nb0=Z*BBi}5gX1mcm2RM%<5iTrb9>cK^4rZbTJ+s7tm1-X=% z;V7~Zn)9XV=Xw+LfYZDQsY_Bb-;(c-T9SQ^>FkxPL(kjzrDBV!_)u&V_TR2)5|dR@ zE?s~6GY>@%SRy<7_FE^~OHz*5Rbm7P5sks^b)w&$cGwjDSPYAVb0!-|2hi>+e9R>u zRcgK_9`M&2K>O60T_Q9RTgSL%y4A^u8bWpt@v*Fsb2G&uXYSxmG zQW@KRI3LpjEhDik@*td|O|WwKJHDPyb2J42V$Sk|UP2r9)PHs4a5 ztcnU+qDtz8VzDLt4Me|PJO#65--fmY_AN>__3+dx zTMYk98>0clL_W!!OGGx`xcX7eMu#G9UoOX!sfMO-T&`VI3>OHbKfL0f7oQ|xa;`rI zeQ)1@3+Son%juB9G&;vkC()1mWrKx^nAi-OCKH z@<$Z>5!Ewqs~Tuv9aon->m_DX>`26@Ck>9DA~@Nk*6f{qW1Xo^-Rx8yL731Lec{}t zutbKfG-xNzXbWU5&nkW_FrrqsDd6t=lsYD{30b08wF=bPt8`lCx764<&Y!jP_ZcjY z;2b0)dR!klk>b^Um^$911bJZOcdsOy8f=Qa{Y6!w%RN^thzqF;d~2C3Za9;4*T&ux zL{7&hI~X&vfQMgcx2UK#ehy~1@|GMrpet3q=#OSx=+FtuNP0>m^S$+lbUi7D+u^tg z*p;0#h`atFbEtq?G2hw&F_>6$A<$QFPnQi5_Cu{Og5P;O<62MX$m0#el>5--5qw=b zv_y(yBJDTB02sX~sC<=j2eDoC)DzzvN3Mr_;=sE+M`zGFpTpEcPkXrGxnxbQDP0VI z5c2xh*Lp^#>I&eW3>FR4UQWp&E?mKcSa@2@Cg|RmI-I`vDGVfrqKL)f&=TX;){`{q z)T#c!65hEx%9cK-e0+9BQQWqD`(Ky2m*E=^#2Rhq$(khgm-hvqQ7Ov084ljfDKl4@K~kncz~Y_`!I>I;jH5pH6{CG`aYhci8M09SxK(Ta#&* z7>SQ^?3ozS;Q1_VMu;}Za>}~sLe2b-k0~I~(U~=zkFL&P2>#LgAw;rc@OeJPe$)}B z$)VVf9=03gX^CQKNyD}o5MEf+C_#@tg{qV)*dciIF;Za@LN4P0oE=BW(UG3#JE1RZ zl|e(giOln<8^<17FGg1kGmEl_oL;1ayc%JRdeVS24jGMF0#%_N>Wk@wQf$x8lX)SoFZhVy{Om3&D3u<$nUexe3?$azD!oq7c*{R9 z@v@O%rG6)e9B*A@<7_@7-danhtpa+xZ@$Nfn6ob~DdETa{n%cis7;jI%!^?&kI&0- zU=!F|QSbU(bF~&{_nkoLJ5_3INBAE<%RB_hQTyl#{sd73U5oa>;@2AJjT9;sMv z(nTrLYdv-#N@8{Do~zuc?@zO-G)#k%T}&M2iN+)|$Bb=N!zUQNM2%NG>3_@#aJ=l! zrTGY#5YywSVUxg()I8geV)R33b>0^GWQaJr)2UWEvg)T{L|4w0-w1NweGqCC1naP| zVRsUJW;=#8xHa@_{z>%84d-rXKU(T-qSp*$tN@zsa>2c6N%2Yn-NO=YV8z(+P}f$m zV|}>G-m8BimEpRYem)D<(eY;KxJBQJivRq#wV3wwd7s5|3Xe%Jz+s$s03q z5XX@T#lr2|!Q;APF-~$8xMZSeb;sFqivMnV-;}xj>3XN3mDu3MQ1pWRQL9@}MGA~a zYCa|lX(H;@HvPv83~*sJc7oIoJy#u$`y4A!D~b>5^4n5IOP)AShy&c_6Rru6jGHHz6; z3n!#|XR3pj7zLJxF<$^O37QAaLBucpHBoXSD~7mV5Jp1I?|WQSZ;md%w0O3-xm@XY zWiXoHufZ0kzvkgaoJ$Q=8=4c9-SA!VQ&yU8Uz$W^$UeM@g1k`T!8F0ND6*roeK&iS z{)>wdn$Bvxsj_glO7bU{jwNm9A|=E#*xwQ#ksXYu(2)?TA~XDAVstS$Zwsd2K}0Se zcggRt2_W=?Z7|mOFc|4$OHB`>3r}RsT6f(iiq6=7?@VPN$ZuuEh}LDm){LjpRG+xq;f$%i55caQfq5K!Z!sGB?I zUzc*PmOL$<5d{Uf51l?t_Jl_=q7r%`>#k>~&vktP1Gi?$(Gfxl&N2zQp5@lX*nRya zY~ExErsKf8(#{GiJxJJkK8f)cA-E!v%^`GivQC{VnT9yv(+T5LUe)HX5FdjC>OJmCP7e4WA z%`bNt{Mpyma+Bntfa{PL40MAt=9HXK12J*W8;tyJ@7Nm6v6kSZ4t9oa&7t=h7_Eg()dgY~7Tphkxz)tuX`q(Crf0#hlrdk(bfiF&yQo`-eg$=wnE}H55607uNtA6B%2d5!h3z_PD+E>)&53ap5 zxq<2;HX5g+S~6lxgZM9#I_An1^YW#MH!6@o4V_1{6btKP{K#}(M@;dNT&o9Ik*y)y zJF;siEo&^}(3Sy*mAAqEw>5n)?pcqBhpG|f}^Ha7%O9%mau25Xir_F51b-&>oYCQqN5n~@Q+=$t1EO5hd-rlZDE zzSN^Q?d_mC;1*@VXJ{@k&Xmaic;}95*0B4$ngQ_~tag1;)A#cj(wrIFJM_-gdN z+uxVm$0!bVU30?F*CFk*p9`pS$&?|e^TAx9e*Ot32{kAoWXc~OB9Kn^Qkk5Sj2%Bf zJT9mW2m*j%49WIzJwo)Wxpf(eW$K|yW6f4{_VLnnM`&g(Bxd_-z9H&~V&j_o1wVhYn7!T1&uZDNw|Gw<-Jj z?kcj){R%gQsFed(bye1{D~G-sYK8g$L{gB1eC}|LIL)A6boyztc!n~&x(h}zhV1#T y));^vZ!Oh-T!0|Jw=(_*7rwWC?EmQEBg+8=Q5)vvYVp>e6&IEfDihH8{@(y?Ob*2W diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index d56109c..0b2d554 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -10,7 +10,7 @@ (defn freeze-reader [x] (binding [*print-dup* false] (pr-str x))) (defn thaw-reader [x] (binding [*read-eval* false] (read-string x))) -(def roundtrip-reader (comp freeze-reader thaw-reader)) +(def roundtrip-reader (comp thaw-reader freeze-reader)) (def roundtrip-defaults (comp thaw freeze)) (def roundtrip-encrypted (comp #(thaw % {:password [:cached "p"]}) @@ -65,6 +65,12 @@ (println "Done! (Time for cake?)")) + ;;; 13 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha1 + ;; {:reader {:freeze 23124, :thaw 26469, :round 47674, :data-size 22923}} + ;; {:defaults {:freeze 4007, :thaw 2520, :round 6038, :data-size 12387}} + ;; {:encrypted {:freeze 5560, :thaw 3867, :round 9157, :data-size 12405}} + ;; {:fast {:freeze 3429, :thaw 2078, :round 5577, :data-size 13237}} + ;;; 11 June 2013: Clojure 1.5.1, Nippy 1.3.0-alpha1 ;; {:reader {:freeze 17042, :thaw 31579, :round 48379, :data-size 22954}} ;; {:fast {:freeze 3078, :thaw 4684, :round 8117, :data-size 13274}}