diff --git a/README.md b/README.md index 3a70870..aaa93ce 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version: ```clojure [com.taoensso/nippy "1.2.1"] ; Stable -[com.taoensso/nippy "2.0.0-alpha5"] ; Development (notes below) +[com.taoensso/nippy "2.0.0-alpha6"] ; Development (notes below) ``` 2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the old `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`. **PLEASE REPORT ANY PROBLEMS!** @@ -137,4 +137,4 @@ Otherwise reach me (Peter Taoussanis) at [taoensso.com](https://www.taoensso.com ## License -Copyright © 2012, 2013 Peter Taoussanis. Distributed under the [Eclipse Public License](http://www.eclipse.org/legal/epl-v10.html), the same as Clojure. \ No newline at end of file +Copyright © 2012, 2013 Peter Taoussanis. Distributed under the [Eclipse Public License](http://www.eclipse.org/legal/epl-v10.html), the same as Clojure. diff --git a/benchmarks/chart.png b/benchmarks/chart.png index 8fbbff7..32b4d2d 100644 Binary files a/benchmarks/chart.png and b/benchmarks/chart.png differ diff --git a/project.clj b/project.clj index d73a99b..5d7eac0 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.0.0-alpha5" +(defproject com.taoensso/nippy "2.0.0-alpha6" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License" @@ -20,4 +20,4 @@ [lein-autoexpect "0.2.5"] [codox "0.6.4"]] :min-lein-version "2.0.0" - :warn-on-reflection true) \ No newline at end of file + :warn-on-reflection true) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index abd87db..32928ec 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -65,49 +65,26 @@ (def ^:const id-old-map (int 22)) ; as of 0.9.0, for more efficient thaw (def ^:const id-old-keyword (int 12)) ; as of 2.0.0-alpha5, for str consistecy -;;;; Shared low-level stream stuff - -(defn- write-id [^DataOutputStream stream ^Integer id] (.writeByte stream id)) - -(defn- write-bytes - [^DataOutputStream stream ^bytes ba] - (let [size (alength ba)] - (.writeInt stream size) - (.write stream ba 0 size))) - -(defn- write-biginteger - [^DataOutputStream stream ^BigInteger x] - (write-bytes stream (.toByteArray x))) - -(defn- write-utf8 - [^DataOutputStream stream ^String x] - (write-bytes stream (.getBytes x "UTF-8"))) - -(defn- read-bytes - ^bytes [^DataInputStream stream] - (let [size (.readInt stream) - ba (byte-array size)] - (.read stream ba 0 size) ba)) - -(defn- read-biginteger - ^BigInteger [^DataInputStream stream] - (BigInteger. (read-bytes stream))) - -(defn- read-utf8 - [^DataInputStream stream] - (String. (read-bytes stream) "UTF-8")) - ;;;; Freezing - (defprotocol Freezable (freeze-to-stream* [this stream])) -(defn- freeze-to-stream +(defmacro ^:private write-id [s id] `(.writeByte ~s ~id)) +(defmacro ^:private write-bytes [s ba] + `(let [s# ~s ba# ~ba] + (let [size# (alength ba#)] + (.writeInt s# size#) + (.write s# ba# 0 size#)))) + +(defmacro ^:private write-biginteger [s x] `(write-bytes ~s (.toByteArray ~x))) +(defmacro ^:private write-utf8 [s x] `(write-bytes ~s (.getBytes ~x "UTF-8"))) +(defmacro ^:private freeze-to-stream "Like `freeze-to-stream*` but with metadata support." - [x ^DataOutputStream s] - (if-let [m (meta x)] - (do (write-id s id-meta) - (freeze-to-stream m s))) - (freeze-to-stream* x s)) + [x s] + `(let [x# ~x s# ~s] + (if-let [m# (meta x#)] + (do (write-id s# ~id-meta) + (freeze-to-stream* m# s#))) + (freeze-to-stream* x# s#))) (defmacro ^:private freezer "Helper to extend Freezable protocol." @@ -134,7 +111,7 @@ (freeze-to-stream k# ~'s) (freeze-to-stream v# ~'s)))) -(freezer (Class/forName "[B") id-bytes (write-bytes s x)) +(freezer (Class/forName "[B") id-bytes (write-bytes s ^bytes x)) (freezer nil id-nil) (freezer Boolean id-boolean (.writeBoolean s x)) @@ -206,16 +183,25 @@ (declare thaw-from-stream) -(defn coll-thaw - "Thaws simple collection types." - [coll ^DataInputStream s] - (utils/repeatedly-into coll (.readInt s) #(thaw-from-stream s))) +(defmacro ^:private read-bytes [s] + `(let [s# ~s + size# (.readInt s#) + ba# (byte-array size#)] + (.read s# ba# 0 size#) ba#)) -(defn coll-thaw-kvs - "Thaws key-value collection types." - [coll ^DataInputStream s] - (utils/repeatedly-into coll (/ (.readInt s) 2) - (fn [] [(thaw-from-stream s) (thaw-from-stream s)]))) +(defmacro ^:private read-biginteger [s] `(BigInteger. (read-bytes ~s))) +(defmacro ^:private read-utf8 [s] `(String. (read-bytes ~s) "UTF-8")) + +(defmacro ^:private coll-thaw "Thaws simple collection types." + [s coll] + `(let [s# ~s] + (utils/repeatedly-into ~coll (.readInt s#) (thaw-from-stream s#)))) + +(defmacro ^:private coll-thaw-kvs "Thaws key-value collection types." + [s coll] + `(let [s# ~s] + (utils/repeatedly-into ~coll (/ (.readInt s#) 2) + [(thaw-from-stream s#) (thaw-from-stream s#)]))) (defn- thaw-from-stream [^DataInputStream s] @@ -223,7 +209,7 @@ (utils/case-eval type-id - id-reader (read-string (String. (read-bytes s) "UTF-8")) + id-reader (read-string (read-utf8 s)) id-bytes (read-bytes s) id-nil nil id-boolean (.readBoolean s) @@ -232,15 +218,15 @@ id-string (read-utf8 s) id-keyword (keyword (read-utf8 s)) - id-queue (coll-thaw (PersistentQueue/EMPTY) s) - id-sorted-set (coll-thaw (sorted-set) s) - id-sorted-map (coll-thaw-kvs (sorted-map) s) + id-queue (coll-thaw s (PersistentQueue/EMPTY)) + id-sorted-set (coll-thaw s (sorted-set)) + id-sorted-map (coll-thaw-kvs s (sorted-map)) - id-list (into '() (rseq (coll-thaw [] s))) - id-vector (coll-thaw [] s) - id-set (coll-thaw #{} s) - id-map (coll-thaw-kvs {} s) - id-coll (seq (coll-thaw [] s)) + id-list (into '() (rseq (coll-thaw s []))) + id-vector (coll-thaw s []) + id-set (coll-thaw s #{}) + id-map (coll-thaw-kvs s {}) + id-coll (seq (coll-thaw s [])) id-meta (let [m (thaw-from-stream s)] (with-meta (thaw-from-stream s) m)) @@ -260,8 +246,8 @@ ;;; DEPRECATED id-old-reader (read-string (.readUTF s)) id-old-string (.readUTF s) - id-old-map (apply hash-map (utils/repeatedly-into [] (* 2 (.readInt s)) - #(thaw-from-stream s))) + id-old-map (apply hash-map (utils/repeatedly-into [] + (* 2 (.readInt s)) (thaw-from-stream s))) id-old-keyword (keyword (.readUTF s)) (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) @@ -272,6 +258,9 @@ (when (utils/ba= head-sig* head-sig) [data-ba (head-meta meta-id {:unrecognized-header? true})])))) + +(defn throw-thaw-ex [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e))) + (defn thaw "Deserializes frozen bytes to their original Clojure data type. @@ -291,8 +280,7 @@ compressor compression/default-snappy-compressor encryptor encryption/default-aes128-encryptor}}]] - (let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e))) - try-thaw-data + (let [try-thaw-data (fn [data-ba {decompress? :compressed? decrypt? :encrypted? :or {decompress? compressor decrypt? password} @@ -305,11 +293,13 @@ stream (DataInputStream. (ByteArrayInputStream. ba))] (binding [*read-eval* read-eval?] (thaw-from-stream stream))) (catch Exception e - (cond decrypt? (ex "Wrong password/encryptor?" e) - decompress? (ex "Encrypted data or wrong compressor?" e) - :else (if apparent-header? - (ex "Corrupt data?" e) - (ex "Encrypted and/or compressed data?" e)))))))] + (cond + decrypt? (throw-thaw-ex "Wrong password/encryptor?" e) + decompress? (throw-thaw-ex "Encrypted data or wrong compressor?" e) + :else + (if apparent-header? + (throw-thaw-ex "Corrupt data?" e) + (throw-thaw-ex "Encrypted and/or compressed data?" e)))))))] (if (= legacy-mode true) (try-thaw-data ba nil) @@ -324,25 +314,27 @@ (cond ; Trust metadata, give fancy error messages unrecognized-header? - (ex "Unrecognized header. Data frozen with newer Nippy version?") + (throw-thaw-ex + "Unrecognized header. Data frozen with newer Nippy version?") (and strict? (not encrypted?) password) - (ex (str "Unencrypted data. Try again w/o password.\n" - "Disable `:strict?` option to ignore this error. ")) + (throw-thaw-ex (str "Unencrypted data. Try again w/o password.\n" + "Disable `:strict?` option to ignore this error. ")) (and strict? (not compressed?) compressor) - (ex (str "Uncompressed data. Try again w/o compressor.\n" - "Disable `:strict?` option to ignore this error.")) + (throw-thaw-ex (str "Uncompressed data. Try again w/o compressor.\n" + "Disable `:strict?` option to ignore this error.")) (and compressed? (not compressor)) - (ex "Compressed data. Try again with compressor.") + (throw-thaw-ex "Compressed data. Try again with compressor.") (and encrypted? (not password)) - (ex "Encrypted data. Try again with password.") + (throw-thaw-ex "Encrypted data. Try again with password.") :else (try-thaw-data data-ba head-meta))) ;; Header definitely not okay (if (= legacy-mode :auto) (try-thaw-data ba nil) ; Legacy thaw - (ex (str "Not Nippy data, data frozen with Nippy < 2.x, " - "or corrupt data?\n" - "See `:legacy-mode` option for data frozen with Nippy < 2.x."))))))) + (throw-thaw-ex + (str "Not Nippy data, data frozen with Nippy < 2.x, " + "or corrupt data?\n" + "See `:legacy-mode` option for data frozen with Nippy < 2.x."))))))) (comment (thaw (freeze "hello")) (thaw (freeze "hello" {:compressor nil})) @@ -416,4 +408,4 @@ (thaw ba {:read-eval? read-eval? :compressor (when compressed? compression/default-snappy-compressor) :password password - :legacy-mode true})) \ No newline at end of file + :legacy-mode true})) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 41beb12..dbe72f3 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -34,36 +34,42 @@ (println {:reader - {:freeze (bench (freeze-reader data)) + {:round (bench (roundtrip-reader data)) + :freeze (bench (freeze-reader data)) :thaw (let [frozen (freeze-reader data)] (bench (thaw-reader frozen))) - :round (bench (roundtrip-reader data)) :data-size (count (.getBytes ^String (freeze-reader data) "UTF-8"))}}) (println {:defaults - {:freeze (bench (freeze data)) + {:round (bench (roundtrip-defaults data)) + :freeze (bench (freeze data)) :thaw (let [frozen (freeze data)] (bench (thaw frozen))) - :round (bench (roundtrip-defaults data)) :data-size (count (freeze data))}}) (println {:encrypted - {:freeze (bench (freeze data {:password [:cached "p"]})) + {:round (bench (roundtrip-encrypted data)) + :freeze (bench (freeze data {:password [:cached "p"]})) :thaw (let [frozen (freeze data {:password [:cached "p"]})] (bench (thaw frozen {:password [:cached "p"]}))) - :round (bench (roundtrip-encrypted data)) :data-size (count (freeze data {:password [:cached "p"]}))}}) (println {:fast - {:freeze (bench (freeze data {:compressor nil})) + {:round (bench (roundtrip-fast data)) + :freeze (bench (freeze data {:compressor nil})) :thaw (let [frozen (freeze data {:compressor nil})] (bench (thaw frozen))) - :round (bench (roundtrip-fast data)) :data-size (count (freeze data {:compressor nil}))}}) (println "Done! (Time for cake?)")) + ;;; 16 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha6 + ;; {:reader {:freeze 23601, :thaw 26247, :round 49819, :data-size 22966}} + ;; {:defaults {:freeze 3554, :thaw 2002, :round 5831, :data-size 12394}} + ;; {:encrypted {:freeze 5117, :thaw 3600, :round 9006, :data-size 12420}} + ;; {:fast {:freeze 3247, :thaw 1914, :round 5329, :data-size 13342}} + ;;; 13 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha1 ;; {:reader {:freeze 23124, :thaw 26469, :round 47674, :data-size 22923}} ;; {:defaults {:freeze 4007, :thaw 2520, :round 6038, :data-size 12387}} diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 04ab599..ce57780 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -13,18 +13,23 @@ clauses) ~(when default default)))) -(defn repeatedly-into +(defmacro repeatedly-into "Like `repeatedly` but faster and `conj`s items into given collection." - [coll n f] - (if-not (instance? clojure.lang.IEditableCollection coll) - (loop [v coll idx 0] - (if (>= idx n) - v - (recur (conj v (f)) (inc idx)))) - (loop [v (transient coll) idx 0] - (if (>= idx n) - (persistent! v) - (recur (conj! v (f)) (inc idx)))))) + [coll n & body] + `(let [coll# ~coll + n# ~n] + (if (instance? clojure.lang.IEditableCollection coll#) + (loop [v# (transient coll#) idx# 0] + (if (>= idx# n#) + (persistent! v#) + (recur (conj! v# ~@body) + (inc idx#)))) + (loop [v# coll# + idx# 0] + (if (>= idx# n#) + v# + (recur (conj v# ~@body) + (inc idx#))))))) (defmacro time-ns "Returns number of nanoseconds it takes to execute body." [& body] `(let [t0# (System/nanoTime)] ~@body (- (System/nanoTime) t0#))) @@ -87,4 +92,4 @@ (comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar"))) (let [[x y] (ba-split (.getBytes "foobar") 5)] - [(String. x) (String. y)])) \ No newline at end of file + [(String. x) (String. y)]))