From 5492c1ea0f3d2defbe9d514f05a9f1d11856001b Mon Sep 17 00:00:00 2001 From: Zach Tellman Date: Wed, 9 Oct 2013 23:54:02 -0400 Subject: [PATCH 1/3] don't iterate twice over uncounted seqs, and use explicit key/val accessors for kv-collections, appears to give ~25% improvement in 'freeze' --- src/taoensso/nippy.clj | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index c4c8f83..ce55eac 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -114,17 +114,30 @@ "Extends Freezable to simple collection types." [type id & body] `(freezer ~type ~id - (.writeInt ~'s (count ~'x)) - (doseq [i# ~'x] (freeze-to-stream ~'s i#)))) + (if (counted? ~'x) + (do + (.writeInt ~'s (count ~'x)) + (doseq [i# ~'x] (freeze-to-stream ~'s i#))) + (let [bas# (ByteArrayOutputStream.) + s# (DataOutputStream. bas#) + cnt# (reduce + (fn [cnt# i#] + (freeze-to-stream! s# i#) + (unchecked-inc cnt#)) + 0 + ~'x) + ba# (.toByteArray bas#)] + (.writeInt ~'s cnt#) + (.write ~'s ba# 0 (alength ba#)))))) (defmacro ^:private kv-freezer "Extends Freezable to key-value collection types." [type id & body] `(freezer ~type ~id (.writeInt ~'s (* 2 (count ~'x))) - (doseq [[k# v#] ~'x] - (freeze-to-stream ~'s k#) - (freeze-to-stream ~'s v#)))) + (doseq [kv# ~'x] + (freeze-to-stream ~'s (key kv#)) + (freeze-to-stream ~'s (val kv#))))) (freezer (Class/forName "[B") id-bytes (write-bytes s ^bytes x)) (freezer nil id-nil) From a92c493375d95996b999eed0736e28016bd94814 Mon Sep 17 00:00:00 2001 From: Zach Tellman Date: Thu, 10 Oct 2013 11:31:52 -0400 Subject: [PATCH 2/3] add fast-path encoding for long, doubles, keywords, and strings --- src/taoensso/nippy.clj | 51 +++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index ce55eac..298cc71 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -87,14 +87,29 @@ (defmacro ^:private write-biginteger [s x] `(write-bytes ~s (.toByteArray ~x))) (defmacro ^:private write-utf8 [s x] `(write-bytes ~s (.getBytes ~x "UTF-8"))) -(defmacro ^:private freeze-to-stream - "Like `freeze-to-stream*` but with metadata support." - [s x] - `(let [x# ~x s# ~s] - (when-let [m# (meta x#)] - (write-id s# ~id-meta) - (freeze-to-stream* m# s#)) - (freeze-to-stream* x# s#))) + +(defn- freeze-to-stream + "Like `freeze-to-stream*`, but with metadata support and certain fast-paths encoded." + [^DataOutputStream s x] + (condp instance? x + Number (condp instance? x + Long (do (write-id s id-long) (.writeLong s x)) + Double (do (write-id s id-double) (.writeDouble s x)) + (freeze-to-stream* x s)) + Keyword (do + (write-id s id-keyword) + (write-utf8 s + (if-let [ns (namespace x)] + (str ns "/" (name x)) + (name x)))) + String (do + (write-id s id-string) + (write-utf8 s ^String x)) + (do + (when-let [m (meta x)] + (write-id s id-meta) + (freeze-to-stream* m s)) + (freeze-to-stream* x s)))) (defn freeze-to-stream! "Low-level API. Serializes arg (any Clojure data type) to a DataOutputStream." @@ -120,12 +135,12 @@ (doseq [i# ~'x] (freeze-to-stream ~'s i#))) (let [bas# (ByteArrayOutputStream.) s# (DataOutputStream. bas#) - cnt# (reduce - (fn [cnt# i#] - (freeze-to-stream! s# i#) - (unchecked-inc cnt#)) - 0 - ~'x) + cnt# (loop [cnt# 0, x# ~'x] + (if (empty? x#) + cnt# + (do + (freeze-to-stream! s# (first x#)) + (recur (unchecked-inc cnt#) (rest x#))))) ba# (.toByteArray bas#)] (.writeInt ~'s cnt#) (.write ~'s ba# 0 (alength ba#)))))) @@ -134,10 +149,10 @@ "Extends Freezable to key-value collection types." [type id & body] `(freezer ~type ~id - (.writeInt ~'s (* 2 (count ~'x))) - (doseq [kv# ~'x] - (freeze-to-stream ~'s (key kv#)) - (freeze-to-stream ~'s (val kv#))))) + (.writeInt ~'s (* 2 (count ~'x))) + (doseq [kv# ~'x] + (freeze-to-stream ~'s (key kv#)) + (freeze-to-stream ~'s (val kv#))))) (freezer (Class/forName "[B") id-bytes (write-bytes s ^bytes x)) (freezer nil id-nil) From ae0a2e7907fca0175717ff563a83bcde976c9fb4 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Fri, 11 Oct 2013 15:07:11 +0700 Subject: [PATCH 3/3] Update benchmarks --- src/taoensso/nippy/benchmarks.clj | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index ac0238e..eeff3a9 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -66,6 +66,23 @@ ;; (bench {:reader? false :laps 1}) ;; (bench {:reader? false :laps 2}) + ;;; 11 Oct 2013: Nippy v2.2.0, with both ztellman mods + ;; {:defaults {:round 4319, :freeze 2950, :thaw 1446, :data-size 12369}} + ;; {:encrypted {:round 7675, :freeze 4479, :thaw 3160, :data-size 12388}} + ;; {:fast {:round 3928, :freeze 2530, :thaw 1269, :data-size 13277}} + ;; {:defaults-delta {:round 0.84 :freeze 0.79 :thaw 1.14}} ; vs 2.2.0 + + ;;; 11 Oct 2013: Nippy v2.2.0, with first ztellman mod + ;; {:defaults {:round 4059, :freeze 2578, :thaw 1351, :data-size 12342}} + ;; {:encrypted {:round 7248, :freeze 4058, :thaw 3041, :data-size 12372}} + ;; {:fast {:round 3430, :freeze 2085, :thaw 1229, :data-size 13277}} + ;; {:defaults-delta {:round 0.79 :freeze 0.69 :thaw 1.07}} ; vs 2.2.0 + + ;;; 11 Oct 2013: Nippy v2.2.0 + ;; {:defaults {:round 5135, :freeze 3711, :thaw 1266, :data-size 12393}} + ;; {:encrypted {:round 8655, :freeze 5323, :thaw 3036, :data-size 12420}} + ;; {:fast {:round 4670, :freeze 3282, :thaw 1294, :data-size 13277}} + ;;; 7 Auguest 2013: Nippy v2.2.0-RC1 ;; {:reader {:round 71582, :freeze 13656, :thaw 56730, :data-size 22964}} ;; {:defaults {:round 5619, :freeze 3710, :thaw 1783, :data-size 12368}}