luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev'

Browse source
This commit is contained in:
Peter Taoussanis 2013-06-16 14:08:31 +07:00
commit da394753ce
6 changed files with 105 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version:
```clojure
[com.taoensso/nippy "1.2.1"] ; Stable
[com.taoensso/nippy "2.0.0-alpha5"] ; Development (notes below)
[com.taoensso/nippy "2.0.0-alpha6"] ; Development (notes below)
```
2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the old `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`. **PLEASE REPORT ANY PROBLEMS!**
@ -137,4 +137,4 @@ Otherwise reach me (Peter Taoussanis) at [taoensso.com](https://www.taoensso.com
## License
Copyright © 2012, 2013 Peter Taoussanis. Distributed under the [Eclipse Public License](http://www.eclipse.org/legal/epl-v10.html), the same as Clojure.
Copyright © 2012, 2013 Peter Taoussanis. Distributed under the [Eclipse Public License](http://www.eclipse.org/legal/epl-v10.html), the same as Clojure.

BIN
benchmarks/chart.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 21 KiB

4
project.clj
View file

@ -1,4 +1,4 @@
(defproject com.taoensso/nippy "2.0.0-alpha5"
(defproject com.taoensso/nippy "2.0.0-alpha6"
:description "Clojure serialization library"
:url "https://github.com/ptaoussanis/nippy"
:license {:name "Eclipse Public License"
@ -20,4 +20,4 @@
[lein-autoexpect "0.2.5"]
[codox "0.6.4"]]
:min-lein-version "2.0.0"
:warn-on-reflection true)
:warn-on-reflection true)

148
src/taoensso/nippy.clj
View file

@ -65,49 +65,26 @@
(def ^:const id-old-map (int 22)) ; as of 0.9.0, for more efficient thaw
(def ^:const id-old-keyword (int 12)) ; as of 2.0.0-alpha5, for str consistecy
;;;; Shared low-level stream stuff
(defn- write-id [^DataOutputStream stream ^Integer id] (.writeByte stream id))
(defn- write-bytes
[^DataOutputStream stream ^bytes ba]
(let [size (alength ba)]
(.writeInt stream size)
(.write stream ba 0 size)))
(defn- write-biginteger
[^DataOutputStream stream ^BigInteger x]
(write-bytes stream (.toByteArray x)))
(defn- write-utf8
[^DataOutputStream stream ^String x]
(write-bytes stream (.getBytes x "UTF-8")))
(defn- read-bytes
^bytes [^DataInputStream stream]
(let [size (.readInt stream)
ba (byte-array size)]
(.read stream ba 0 size) ba))
(defn- read-biginteger
^BigInteger [^DataInputStream stream]
(BigInteger. (read-bytes stream)))
(defn- read-utf8
[^DataInputStream stream]
(String. (read-bytes stream) "UTF-8"))
;;;; Freezing
(defprotocol Freezable (freeze-to-stream* [this stream]))
(defn- freeze-to-stream
(defmacro ^:private write-id [s id] `(.writeByte ~s ~id))
(defmacro ^:private write-bytes [s ba]
`(let [s# ~s ba# ~ba]
(let [size# (alength ba#)]
(.writeInt s# size#)
(.write s# ba# 0 size#))))
(defmacro ^:private write-biginteger [s x] `(write-bytes ~s (.toByteArray ~x)))
(defmacro ^:private write-utf8 [s x] `(write-bytes ~s (.getBytes ~x "UTF-8")))
(defmacro ^:private freeze-to-stream
"Like `freeze-to-stream*` but with metadata support."
[x ^DataOutputStream s]
(if-let [m (meta x)]
(do (write-id s id-meta)
(freeze-to-stream m s)))
(freeze-to-stream* x s))
[x s]
`(let [x# ~x s# ~s]
(if-let [m# (meta x#)]
(do (write-id s# ~id-meta)
(freeze-to-stream* m# s#)))
(freeze-to-stream* x# s#)))
(defmacro ^:private freezer
"Helper to extend Freezable protocol."
@ -134,7 +111,7 @@
(freeze-to-stream k# ~'s)
(freeze-to-stream v# ~'s))))
(freezer (Class/forName "[B") id-bytes (write-bytes s x))
(freezer (Class/forName "[B") id-bytes (write-bytes s ^bytes x))
(freezer nil id-nil)
(freezer Boolean id-boolean (.writeBoolean s x))
@ -206,16 +183,25 @@
(declare thaw-from-stream)
(defn coll-thaw
"Thaws simple collection types."
[coll ^DataInputStream s]
(utils/repeatedly-into coll (.readInt s) #(thaw-from-stream s)))
(defmacro ^:private read-bytes [s]
`(let [s# ~s
size# (.readInt s#)
ba# (byte-array size#)]
(.read s# ba# 0 size#) ba#))
(defn coll-thaw-kvs
"Thaws key-value collection types."
[coll ^DataInputStream s]
(utils/repeatedly-into coll (/ (.readInt s) 2)
(fn [] [(thaw-from-stream s) (thaw-from-stream s)])))
(defmacro ^:private read-biginteger [s] `(BigInteger. (read-bytes ~s)))
(defmacro ^:private read-utf8 [s] `(String. (read-bytes ~s) "UTF-8"))
(defmacro ^:private coll-thaw "Thaws simple collection types."
[s coll]
`(let [s# ~s]
(utils/repeatedly-into ~coll (.readInt s#) (thaw-from-stream s#))))
(defmacro ^:private coll-thaw-kvs "Thaws key-value collection types."
[s coll]
`(let [s# ~s]
(utils/repeatedly-into ~coll (/ (.readInt s#) 2)
[(thaw-from-stream s#) (thaw-from-stream s#)])))
(defn- thaw-from-stream
[^DataInputStream s]
@ -223,7 +209,7 @@
(utils/case-eval
type-id
id-reader (read-string (String. (read-bytes s) "UTF-8"))
id-reader (read-string (read-utf8 s))
id-bytes (read-bytes s)
id-nil nil
id-boolean (.readBoolean s)
@ -232,15 +218,15 @@
id-string (read-utf8 s)
id-keyword (keyword (read-utf8 s))
id-queue (coll-thaw (PersistentQueue/EMPTY) s)
id-sorted-set (coll-thaw (sorted-set) s)
id-sorted-map (coll-thaw-kvs (sorted-map) s)
id-queue (coll-thaw s (PersistentQueue/EMPTY))
id-sorted-set (coll-thaw s (sorted-set))
id-sorted-map (coll-thaw-kvs s (sorted-map))
id-list (into '() (rseq (coll-thaw [] s)))
id-vector (coll-thaw [] s)
id-set (coll-thaw #{} s)
id-map (coll-thaw-kvs {} s)
id-coll (seq (coll-thaw [] s))
id-list (into '() (rseq (coll-thaw s [])))
id-vector (coll-thaw s [])
id-set (coll-thaw s #{})
id-map (coll-thaw-kvs s {})
id-coll (seq (coll-thaw s []))
id-meta (let [m (thaw-from-stream s)] (with-meta (thaw-from-stream s) m))
@ -260,8 +246,8 @@
;;; DEPRECATED
id-old-reader (read-string (.readUTF s))
id-old-string (.readUTF s)
id-old-map (apply hash-map (utils/repeatedly-into [] (* 2 (.readInt s))
#(thaw-from-stream s)))
id-old-map (apply hash-map (utils/repeatedly-into []
(* 2 (.readInt s)) (thaw-from-stream s)))
id-old-keyword (keyword (.readUTF s))
(throw (Exception. (str "Failed to thaw unknown type ID: " type-id))))))
@ -272,6 +258,9 @@
(when (utils/ba= head-sig* head-sig)
[data-ba (head-meta meta-id {:unrecognized-header? true})]))))
(defn throw-thaw-ex [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
(defn thaw
"Deserializes frozen bytes to their original Clojure data type.
@ -291,8 +280,7 @@
compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]]
(let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
try-thaw-data
(let [try-thaw-data
(fn [data-ba {decompress? :compressed? decrypt? :encrypted?
:or {decompress? compressor
decrypt? password}
@ -305,11 +293,13 @@
stream (DataInputStream. (ByteArrayInputStream. ba))]
(binding [*read-eval* read-eval?] (thaw-from-stream stream)))
(catch Exception e
(cond decrypt? (ex "Wrong password/encryptor?" e)
decompress? (ex "Encrypted data or wrong compressor?" e)
:else (if apparent-header?
(ex "Corrupt data?" e)
(ex "Encrypted and/or compressed data?" e)))))))]
(cond
decrypt? (throw-thaw-ex "Wrong password/encryptor?" e)
decompress? (throw-thaw-ex "Encrypted data or wrong compressor?" e)
:else
(if apparent-header?
(throw-thaw-ex "Corrupt data?" e)
(throw-thaw-ex "Encrypted and/or compressed data?" e)))))))]
(if (= legacy-mode true)
(try-thaw-data ba nil)
@ -324,25 +314,27 @@
(cond ; Trust metadata, give fancy error messages
unrecognized-header?
(ex "Unrecognized header. Data frozen with newer Nippy version?")
(throw-thaw-ex
"Unrecognized header. Data frozen with newer Nippy version?")
(and strict? (not encrypted?) password)
(ex (str "Unencrypted data. Try again w/o password.\n"
"Disable `:strict?` option to ignore this error. "))
(throw-thaw-ex (str "Unencrypted data. Try again w/o password.\n"
"Disable `:strict?` option to ignore this error. "))
(and strict? (not compressed?) compressor)
(ex (str "Uncompressed data. Try again w/o compressor.\n"
"Disable `:strict?` option to ignore this error."))
(throw-thaw-ex (str "Uncompressed data. Try again w/o compressor.\n"
"Disable `:strict?` option to ignore this error."))
(and compressed? (not compressor))
(ex "Compressed data. Try again with compressor.")
(throw-thaw-ex "Compressed data. Try again with compressor.")
(and encrypted? (not password))
(ex "Encrypted data. Try again with password.")
(throw-thaw-ex "Encrypted data. Try again with password.")
:else (try-thaw-data data-ba head-meta)))
;; Header definitely not okay
(if (= legacy-mode :auto)
(try-thaw-data ba nil) ; Legacy thaw
(ex (str "Not Nippy data, data frozen with Nippy < 2.x, "
"or corrupt data?\n"
"See `:legacy-mode` option for data frozen with Nippy < 2.x.")))))))
(throw-thaw-ex
(str "Not Nippy data, data frozen with Nippy < 2.x, "
"or corrupt data?\n"
"See `:legacy-mode` option for data frozen with Nippy < 2.x.")))))))
(comment (thaw (freeze "hello"))
(thaw (freeze "hello" {:compressor nil}))
@ -416,4 +408,4 @@
(thaw ba {:read-eval? read-eval?
:compressor (when compressed? compression/default-snappy-compressor)
:password password
:legacy-mode true}))
:legacy-mode true}))

22
src/taoensso/nippy/benchmarks.clj
View file

@ -34,36 +34,42 @@
(println
{:reader
{:freeze (bench (freeze-reader data))
{:round (bench (roundtrip-reader data))
:freeze (bench (freeze-reader data))
:thaw (let [frozen (freeze-reader data)] (bench (thaw-reader frozen)))
:round (bench (roundtrip-reader data))
:data-size (count (.getBytes ^String (freeze-reader data) "UTF-8"))}})
(println
{:defaults
{:freeze (bench (freeze data))
{:round (bench (roundtrip-defaults data))
:freeze (bench (freeze data))
:thaw (let [frozen (freeze data)] (bench (thaw frozen)))
:round (bench (roundtrip-defaults data))
:data-size (count (freeze data))}})
(println
{:encrypted
{:freeze (bench (freeze data {:password [:cached "p"]}))
{:round (bench (roundtrip-encrypted data))
:freeze (bench (freeze data {:password [:cached "p"]}))
:thaw (let [frozen (freeze data {:password [:cached "p"]})]
(bench (thaw frozen {:password [:cached "p"]})))
:round (bench (roundtrip-encrypted data))
:data-size (count (freeze data {:password [:cached "p"]}))}})
(println
{:fast
{:freeze (bench (freeze data {:compressor nil}))
{:round (bench (roundtrip-fast data))
:freeze (bench (freeze data {:compressor nil}))
:thaw (let [frozen (freeze data {:compressor nil})]
(bench (thaw frozen)))
:round (bench (roundtrip-fast data))
:data-size (count (freeze data {:compressor nil}))}})
(println "Done! (Time for cake?)"))
;;; 16 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha6
;; {:reader {:freeze 23601, :thaw 26247, :round 49819, :data-size 22966}}
;; {:defaults {:freeze 3554, :thaw 2002, :round 5831, :data-size 12394}}
;; {:encrypted {:freeze 5117, :thaw 3600, :round 9006, :data-size 12420}}
;; {:fast {:freeze 3247, :thaw 1914, :round 5329, :data-size 13342}}
;;; 13 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha1
;; {:reader {:freeze 23124, :thaw 26469, :round 47674, :data-size 22923}}
;; {:defaults {:freeze 4007, :thaw 2520, :round 6038, :data-size 12387}}

29
src/taoensso/nippy/utils.clj
View file

@ -13,18 +13,23 @@
clauses)
~(when default default))))
(defn repeatedly-into
(defmacro repeatedly-into
"Like `repeatedly` but faster and `conj`s items into given collection."
[coll n f]
(if-not (instance? clojure.lang.IEditableCollection coll)
(loop [v coll idx 0]
(if (>= idx n)
v
(recur (conj v (f)) (inc idx))))
(loop [v (transient coll) idx 0]
(if (>= idx n)
(persistent! v)
(recur (conj! v (f)) (inc idx))))))
[coll n & body]
`(let [coll# ~coll
n# ~n]
(if (instance? clojure.lang.IEditableCollection coll#)
(loop [v# (transient coll#) idx# 0]
(if (>= idx# n#)
(persistent! v#)
(recur (conj! v# ~@body)
(inc idx#))))
(loop [v# coll#
idx# 0]
(if (>= idx# n#)
v#
(recur (conj v# ~@body)
(inc idx#)))))))
(defmacro time-ns "Returns number of nanoseconds it takes to execute body."
[& body] `(let [t0# (System/nanoTime)] ~@body (- (System/nanoTime) t0#)))
@ -87,4 +92,4 @@
(comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar")))
(let [[x y] (ba-split (.getBytes "foobar") 5)]
[(String. x) (String. y)]))
[(String. x) (String. y)]))