Merge branch 'dev'
This commit is contained in:
Peter Taoussanis
commit
1a92e192c9
5 changed files with 139 additions and 31 deletions
11
CHANGELOG.md
11
CHANGELOG.md
|
|
@ -1,3 +1,14 @@
|
|||
## v2.0.0 → v2.1.0
|
||||
* Exposed low-level fns: `freeze-to-stream!`, `thaw-from-stream!`.
|
||||
* Added `extend-freeze` and `extend-thaw` for extending to custom types:
|
||||
* Added support for easily extending Nippy de/serialization to custom types:
|
||||
```clojure
|
||||
(defrecord MyType [data])
|
||||
(nippy/extend-freeze MyType 1 [x steam] (.writeUTF stream (:data x)))
|
||||
(nippy/extend-thaw 1 [stream] (->MyType (.readUTF stream)))
|
||||
(nippy/thaw (nippy/freeze (->MyType "Joe"))) => #taoensso.nippy.MyType{:data "Joe"}
|
||||
```
|
||||
|
||||
## v1.2.1 → v2.0.0
|
||||
* **MIGRATION NOTE**: Please be sure to use `lein clean` to clear old (v1) build artifacts!
|
||||
* Refactored for huge performance improvements (~40% roundtrip time).
|
||||
|
|
|
|||
25
README.md
25
README.md
|
|
@ -1,7 +1,7 @@
|
|||
**[API docs](http://ptaoussanis.github.io/nippy/)** | **[CHANGELOG](https://github.com/ptaoussanis/nippy/blob/master/CHANGELOG.md)** | [contact & contributing](#contact--contributing) | [other Clojure libs](https://www.taoensso.com/clojure-libraries) | [Twitter](https://twitter.com/#!/ptaoussanis) | current [semantic](http://semver.org/) version:
|
||||
|
||||
```clojure
|
||||
[com.taoensso/nippy "2.0.0"] ; See CHANGELOG for changes since 1.x
|
||||
[com.taoensso/nippy "2.1.0"] ; See CHANGELOG for changes since 1.x
|
||||
```
|
||||
|
||||
v2 adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), easier integration into other tools/libraries, and hugely improved performance.
|
||||
|
|
@ -20,8 +20,9 @@ Nippy is an attempt to provide a reliable, high-performance **drop-in alternativ
|
|||
## What's in the box™?
|
||||
* Small, uncomplicated **all-Clojure** library.
|
||||
* **Great performance**.
|
||||
* Comprehesive, extensible **support for all major data types**.
|
||||
* **Reader-fallback** for difficult/future types (including Clojure 1.4+ tagged literals).
|
||||
* Comprehesive **support for all standard data types**.
|
||||
* **Easily extendable to custom data types**. (v2.1+)
|
||||
* **Reader-fallback** for all other types (including Clojure 1.4+ tagged literals).
|
||||
* **Full test coverage** for every supported type.
|
||||
* Fully pluggable **compression**, including built-in high-performance [Snappy](http://code.google.com/p/snappy/) compressor.
|
||||
* Fully pluggable **encryption**, including built-in high-strength AES128 enabled with a single `:password [:salted "my-password"]` option. (v2+)
|
||||
|
|
@ -34,7 +35,7 @@ Nippy is an attempt to provide a reliable, high-performance **drop-in alternativ
|
|||
Add the necessary dependency to your [Leiningen](http://leiningen.org/) `project.clj` and `require` the library in your ns:
|
||||
|
||||
```clojure
|
||||
[com.taoensso/nippy "2.0.0"] ; project.clj
|
||||
[com.taoensso/nippy "2.1.0"] ; project.clj
|
||||
(ns my-app (:require [taoensso.nippy :as nippy])) ; ns
|
||||
```
|
||||
|
||||
|
|
@ -117,6 +118,22 @@ Nippy v2+ also gives you **dead simple data encryption**. Add a single option to
|
|||
|
||||
There's two default forms of encryption on offer: `:salted` and `:cached`. Each of these makes carefully-chosen trade-offs and is suited to one of two common use cases. See the `aes128-encryptor` [docstring](http://ptaoussanis.github.io/nippy/taoensso.nippy.encryption.html) for a detailed explanation of why/when you'd want one or the other.
|
||||
|
||||
### Custom types (v2.1+, ALPHA - subject to change)
|
||||
|
||||
```clojure
|
||||
(defrecord MyType [data])
|
||||
|
||||
(nippy/extend-freeze MyType 1 ; A unique type id ∈[1, 128]
|
||||
[x data-output-steam]
|
||||
(.writeUTF data-output-stream (:data x)))
|
||||
|
||||
(nippy/extend-thaw 1 ; Same type id
|
||||
[data-input-stream]
|
||||
(->MyType (.readUTF data-input-stream)))
|
||||
|
||||
(nippy/thaw (nippy/freeze (->MyType "Joe"))) => #taoensso.nippy.MyType{:data "Joe"}
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
(defproject com.taoensso/nippy "2.0.0"
|
||||
(defproject com.taoensso/nippy "2.1.0"
|
||||
:description "Clojure serialization library"
|
||||
:url "https://github.com/ptaoussanis/nippy"
|
||||
:license {:name "Eclipse Public License"
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
;;;; Nippy 2.x+ header spec (4 bytes)
|
||||
(def ^:private ^:const head-version 1)
|
||||
(def ^:private head-sig (.getBytes "NPY" "UTF-8"))
|
||||
(def ^:private head-meta "Final byte stores version-dependent metadata."
|
||||
(def ^:private ^:const head-meta "Final byte stores version-dependent metadata."
|
||||
{(byte 0) {:version 1 :compressed? false :encrypted? false}
|
||||
(byte 1) {:version 1 :compressed? true :encrypted? false}
|
||||
(byte 2) {:version 1 :compressed? false :encrypted? true}
|
||||
|
|
@ -23,6 +23,9 @@
|
|||
|
||||
;;;; Data type IDs
|
||||
|
||||
;; **Negative ids reserved for user-defined types**
|
||||
|
||||
(def ^:const id-reserved (int 0))
|
||||
;; 1
|
||||
(def ^:const id-bytes (int 2))
|
||||
(def ^:const id-nil (int 3))
|
||||
|
|
@ -67,7 +70,7 @@
|
|||
;;;; Freezing
|
||||
(defprotocol Freezable (freeze-to-stream* [this stream]))
|
||||
|
||||
(defmacro ^:private write-id [s id] `(.writeByte ~s ~id))
|
||||
(defmacro write-id [s id] `(.writeByte ~s ~id))
|
||||
(defmacro ^:private write-bytes [s ba]
|
||||
`(let [s# ~s ba# ~ba]
|
||||
(let [size# (alength ba#)]
|
||||
|
|
@ -78,18 +81,32 @@
|
|||
(defmacro ^:private write-utf8 [s x] `(write-bytes ~s (.getBytes ~x "UTF-8")))
|
||||
(defmacro ^:private freeze-to-stream
|
||||
"Like `freeze-to-stream*` but with metadata support."
|
||||
[x s]
|
||||
[s x]
|
||||
`(let [x# ~x s# ~s]
|
||||
(if-let [m# (meta x#)]
|
||||
(do (write-id s# ~id-meta)
|
||||
(freeze-to-stream* m# s#)))
|
||||
(freeze-to-stream* x# s#)))
|
||||
(when-let [m# (meta x#)]
|
||||
(write-id s# ~id-meta)
|
||||
(freeze-to-stream* m# s#))
|
||||
(try (freeze-to-stream* x# s#)
|
||||
(catch java.lang.IllegalArgumentException _#
|
||||
;; Use Clojure reader as final fallback (after custom extensions)
|
||||
(write-id s# id-reader)
|
||||
(write-bytes s# (.getBytes (pr-str x#) "UTF-8"))))))
|
||||
|
||||
(defn freeze-to-stream!
|
||||
"Low-level API. Serializes arg (any Clojure data type) to a DataOutputStream."
|
||||
[^DataOutputStream data-output-stream x & [{:keys [print-dup?]
|
||||
:or {print-dup? true}}]]
|
||||
(if (identical? *print-dup* print-dup?)
|
||||
(freeze-to-stream data-output-stream x)
|
||||
(binding [*print-dup* print-dup?] ; Expensive
|
||||
(freeze-to-stream data-output-stream x))))
|
||||
|
||||
|
||||
(defmacro ^:private freezer
|
||||
"Helper to extend Freezable protocol."
|
||||
[type id & body]
|
||||
`(extend-type ~type
|
||||
~'Freezable
|
||||
Freezable
|
||||
(~'freeze-to-stream* [~'x ~(with-meta 's {:tag 'DataOutputStream})]
|
||||
(write-id ~'s ~id)
|
||||
~@body)))
|
||||
|
|
@ -99,7 +116,7 @@
|
|||
[type id & body]
|
||||
`(freezer ~type ~id
|
||||
(.writeInt ~'s (count ~'x))
|
||||
(doseq [i# ~'x] (freeze-to-stream i# ~'s))))
|
||||
(doseq [i# ~'x] (freeze-to-stream ~'s i#))))
|
||||
|
||||
(defmacro ^:private kv-freezer
|
||||
"Extends Freezable to key-value collection types."
|
||||
|
|
@ -107,8 +124,8 @@
|
|||
`(freezer ~type ~id
|
||||
(.writeInt ~'s (* 2 (count ~'x)))
|
||||
(doseq [[k# v#] ~'x]
|
||||
(freeze-to-stream k# ~'s)
|
||||
(freeze-to-stream v# ~'s))))
|
||||
(freeze-to-stream ~'s k#)
|
||||
(freeze-to-stream ~'s v#))))
|
||||
|
||||
(freezer (Class/forName "[B") id-bytes (write-bytes s ^bytes x))
|
||||
(freezer nil id-nil)
|
||||
|
|
@ -147,9 +164,6 @@
|
|||
(write-biginteger s (.numerator x))
|
||||
(write-biginteger s (.denominator x)))
|
||||
|
||||
;; Use Clojure's own reader as final fallback
|
||||
(freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8")))
|
||||
|
||||
(def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta))
|
||||
|
||||
(defn- wrap-header [data-ba metadata]
|
||||
|
|
@ -165,7 +179,8 @@
|
|||
|
||||
(defn freeze
|
||||
"Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to
|
||||
true to produce bytes readble by Nippy < 2.x."
|
||||
true to produce bytes readble by Nippy < 2.x. For custom types extend the
|
||||
Clojure reader or see `extend-freeze`."
|
||||
^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode]
|
||||
:or {print-dup? true
|
||||
compressor snappy-compressor
|
||||
|
|
@ -173,7 +188,7 @@
|
|||
(when legacy-mode (assert-legacy-args compressor password))
|
||||
(let [ba (ByteArrayOutputStream.)
|
||||
stream (DataOutputStream. ba)]
|
||||
(binding [*print-dup* print-dup?] (freeze-to-stream x stream))
|
||||
(freeze-to-stream! stream x {:print-dup? print-dup?})
|
||||
(let [ba (.toByteArray ba)
|
||||
ba (if compressor (compression/compress compressor ba) ba)
|
||||
ba (if password (encryption/encrypt encryptor password ba) ba)]
|
||||
|
|
@ -203,13 +218,14 @@
|
|||
[s coll]
|
||||
`(let [s# ~s]
|
||||
(utils/repeatedly-into ~coll (/ (.readInt s#) 2)
|
||||
[(thaw-from-stream s#) (thaw-from-stream s#)])))
|
||||
[(thaw-from-stream s#) (thaw-from-stream s#)])))
|
||||
|
||||
(declare ^:private custom-readers)
|
||||
|
||||
(defn- thaw-from-stream
|
||||
[^DataInputStream s]
|
||||
(let [type-id (.readByte s)]
|
||||
(utils/case-eval
|
||||
type-id
|
||||
(utils/case-eval type-id
|
||||
|
||||
id-reader (read-string (read-utf8 s))
|
||||
id-bytes (read-bytes s)
|
||||
|
|
@ -252,7 +268,26 @@
|
|||
(* 2 (.readInt s)) (thaw-from-stream s)))
|
||||
id-old-keyword (keyword (.readUTF s))
|
||||
|
||||
(throw (Exception. (str "Failed to thaw unknown type ID: " type-id))))))
|
||||
(if-not (neg? type-id)
|
||||
(throw (Exception. (str "Unknown type ID: " type-id)))
|
||||
|
||||
;; Custom types
|
||||
(if-let [reader (get @custom-readers type-id)]
|
||||
(try (reader s)
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Reader exception for custom type ID: "
|
||||
(- type-id)) e))))
|
||||
(throw (Exception. (str "No reader provided for custom type ID: "
|
||||
(- type-id)))))))))
|
||||
|
||||
(defn thaw-from-stream!
|
||||
"Low-level API. Deserializes a frozen object from given DataInputStream to its
|
||||
original Clojure data type."
|
||||
[data-input-stream & [{:keys [read-eval?]}]]
|
||||
(if (identical? *read-eval* read-eval?)
|
||||
(thaw-from-stream data-input-stream)
|
||||
(binding [*read-eval* read-eval?] ; Expensive
|
||||
(thaw-from-stream data-input-stream))))
|
||||
|
||||
(defn- try-parse-header [ba]
|
||||
(when-let [[head-ba data-ba] (utils/ba-split ba 4)]
|
||||
|
|
@ -261,12 +296,13 @@
|
|||
[data-ba (head-meta meta-id {:unrecognized-header? true})]))))
|
||||
|
||||
(defn thaw
|
||||
"Deserializes frozen bytes to their original Clojure data type. Supports data
|
||||
frozen with current and all previous versions of Nippy.
|
||||
"Deserializes a frozen object from given byte array to its original Clojure
|
||||
data type. Supports data frozen with current and all previous versions of
|
||||
Nippy. For custom types extend the Clojure reader or see `extend-thaw`.
|
||||
|
||||
WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless
|
||||
you are sure you know what you're doing."
|
||||
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-opts]
|
||||
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-opts readers]
|
||||
:or {legacy-opts {:compressed? true}
|
||||
compressor snappy-compressor
|
||||
encryptor aes128-encryptor}
|
||||
|
|
@ -284,7 +320,9 @@
|
|||
ba (if password (encryption/decrypt encryptor password ba) ba)
|
||||
ba (if compressor (compression/decompress compressor ba) ba)
|
||||
stream (DataInputStream. (ByteArrayInputStream. ba))]
|
||||
(binding [*read-eval* read-eval?] (thaw-from-stream stream)))
|
||||
|
||||
(thaw-from-stream! stream {:read-eval? read-eval?}))
|
||||
|
||||
(catch Exception e
|
||||
(cond
|
||||
password (ex "Wrong password/encryptor?" e)
|
||||
|
|
@ -307,7 +345,9 @@
|
|||
:else (try (try-thaw-data data-ba head-meta)
|
||||
(catch Exception e
|
||||
(if legacy-opts
|
||||
(try-thaw-data ba nil)
|
||||
(try (try-thaw-data ba nil)
|
||||
(catch Exception _
|
||||
(throw e)))
|
||||
(throw e)))))
|
||||
|
||||
;; Header definitely not okay
|
||||
|
|
@ -320,6 +360,39 @@
|
|||
(thaw (freeze "hello" {:password [:salted "p"]})) ; ex
|
||||
(thaw (freeze "hello") {:password [:salted "p"]}))
|
||||
|
||||
;;;; Custom types
|
||||
|
||||
(defmacro extend-freeze
|
||||
"Alpha - subject to change.
|
||||
Extends Nippy to support freezing of a custom type with id ∈[1, 128]:
|
||||
(defrecord MyType [data])
|
||||
(extend-freeze MyType 1 [x data-output-stream]
|
||||
(.writeUTF [data-output-stream] (:data x)))"
|
||||
[type custom-type-id [x stream] & body]
|
||||
(assert (and (>= custom-type-id 1) (<= custom-type-id 128)))
|
||||
`(extend-type ~type
|
||||
Freezable
|
||||
(~'freeze-to-stream* [~x ~(with-meta stream {:tag 'java.io.DataOutputStream})]
|
||||
(write-id ~stream ~(int (- custom-type-id)))
|
||||
~@body)))
|
||||
|
||||
(defonce custom-readers (atom {})) ; {<custom-type-id> (fn [data-input-stream]) ...}
|
||||
(defmacro extend-thaw
|
||||
"Alpha - subject to change.
|
||||
Extends Nippy to support thawing of a custom type with id ∈[1, 128]:
|
||||
(extend-thaw 1 [data-input-stream]
|
||||
(->MyType (.readUTF data-input-stream)))"
|
||||
[custom-type-id [stream] & body]
|
||||
(assert (and (>= custom-type-id 1) (<= custom-type-id 128)))
|
||||
`(swap! custom-readers assoc ~(int (- custom-type-id))
|
||||
(fn [~(with-meta stream {:tag 'java.io.DataInputStream})]
|
||||
~@body)))
|
||||
|
||||
(comment (defrecord MyType [data])
|
||||
(extend-freeze MyType 1 [x s] (.writeUTF s (:data x)))
|
||||
(extend-thaw 1 [s] (->MyType (.readUTF s)))
|
||||
(thaw (freeze (->MyType "Joe"))))
|
||||
|
||||
;;;; Stress data
|
||||
|
||||
(def stress-data "Reference data used for tests & benchmarks."
|
||||
|
|
@ -390,4 +463,4 @@
|
|||
:or {compressed? true}}]
|
||||
(thaw ba {:legacy-opts {:compressed? compressed?}
|
||||
:read-eval? read-eval?
|
||||
:password nil}))
|
||||
:password nil}))
|
||||
|
|
@ -30,4 +30,11 @@
|
|||
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
|
||||
|
||||
;;; Custom types
|
||||
(defrecord MyType [data])
|
||||
(nippy/extend-freeze MyType 1 [x s] (.writeUTF s (:data x)))
|
||||
(expect Exception (thaw (freeze (->MyType "Joe"))))
|
||||
(expect (MyType. "Joe") (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s)))
|
||||
(thaw (freeze (->MyType "Joe")))))
|
||||
|
||||
(expect (benchmarks/bench {:reader? false})) ; Also tests :cached passwords
|
||||
Loading…
Reference in a new issue