diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bcf319..4167ef8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v2.0.0 → v2.1.0 + * Exposed low-level fns: `freeze-to-stream!`, `thaw-from-stream!`. + * Added `extend-freeze` and `extend-thaw` for extending to custom types: + * Added support for easily extending Nippy de/serialization to custom types: + ```clojure + (defrecord MyType [data]) + (nippy/extend-freeze MyType 1 [x steam] (.writeUTF stream (:data x))) + (nippy/extend-thaw 1 [stream] (->MyType (.readUTF stream))) + (nippy/thaw (nippy/freeze (->MyType "Joe"))) => #taoensso.nippy.MyType{:data "Joe"} + ``` + ## v1.2.1 → v2.0.0 * **MIGRATION NOTE**: Please be sure to use `lein clean` to clear old (v1) build artifacts! * Refactored for huge performance improvements (~40% roundtrip time). diff --git a/README.md b/README.md index 5994fdc..1b253c6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ **[API docs](http://ptaoussanis.github.io/nippy/)** | **[CHANGELOG](https://github.com/ptaoussanis/nippy/blob/master/CHANGELOG.md)** | [contact & contributing](#contact--contributing) | [other Clojure libs](https://www.taoensso.com/clojure-libraries) | [Twitter](https://twitter.com/#!/ptaoussanis) | current [semantic](http://semver.org/) version: ```clojure -[com.taoensso/nippy "2.0.0"] ; See CHANGELOG for changes since 1.x +[com.taoensso/nippy "2.1.0"] ; See CHANGELOG for changes since 1.x ``` v2 adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), easier integration into other tools/libraries, and hugely improved performance. @@ -20,8 +20,9 @@ Nippy is an attempt to provide a reliable, high-performance **drop-in alternativ ## What's in the box™? * Small, uncomplicated **all-Clojure** library. * **Great performance**. - * Comprehesive, extensible **support for all major data types**. - * **Reader-fallback** for difficult/future types (including Clojure 1.4+ tagged literals). + * Comprehesive **support for all standard data types**. + * **Easily extendable to custom data types**. (v2.1+) + * **Reader-fallback** for all other types (including Clojure 1.4+ tagged literals). * **Full test coverage** for every supported type. * Fully pluggable **compression**, including built-in high-performance [Snappy](http://code.google.com/p/snappy/) compressor. * Fully pluggable **encryption**, including built-in high-strength AES128 enabled with a single `:password [:salted "my-password"]` option. (v2+) @@ -34,7 +35,7 @@ Nippy is an attempt to provide a reliable, high-performance **drop-in alternativ Add the necessary dependency to your [Leiningen](http://leiningen.org/) `project.clj` and `require` the library in your ns: ```clojure -[com.taoensso/nippy "2.0.0"] ; project.clj +[com.taoensso/nippy "2.1.0"] ; project.clj (ns my-app (:require [taoensso.nippy :as nippy])) ; ns ``` @@ -117,6 +118,22 @@ Nippy v2+ also gives you **dead simple data encryption**. Add a single option to There's two default forms of encryption on offer: `:salted` and `:cached`. Each of these makes carefully-chosen trade-offs and is suited to one of two common use cases. See the `aes128-encryptor` [docstring](http://ptaoussanis.github.io/nippy/taoensso.nippy.encryption.html) for a detailed explanation of why/when you'd want one or the other. +### Custom types (v2.1+, ALPHA - subject to change) + +```clojure +(defrecord MyType [data]) + +(nippy/extend-freeze MyType 1 ; A unique type id ∈[1, 128] + [x data-output-steam] + (.writeUTF data-output-stream (:data x))) + +(nippy/extend-thaw 1 ; Same type id + [data-input-stream] + (->MyType (.readUTF data-input-stream))) + +(nippy/thaw (nippy/freeze (->MyType "Joe"))) => #taoensso.nippy.MyType{:data "Joe"} +``` + ## Performance ![Comparison chart](https://github.com/ptaoussanis/nippy/raw/master/benchmarks.png) diff --git a/project.clj b/project.clj index 73f4592..7c22550 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.0.0" +(defproject com.taoensso/nippy "2.1.0" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License" diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index b742f6c..9a6a541 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -15,7 +15,7 @@ ;;;; Nippy 2.x+ header spec (4 bytes) (def ^:private ^:const head-version 1) (def ^:private head-sig (.getBytes "NPY" "UTF-8")) -(def ^:private head-meta "Final byte stores version-dependent metadata." +(def ^:private ^:const head-meta "Final byte stores version-dependent metadata." {(byte 0) {:version 1 :compressed? false :encrypted? false} (byte 1) {:version 1 :compressed? true :encrypted? false} (byte 2) {:version 1 :compressed? false :encrypted? true} @@ -23,6 +23,9 @@ ;;;; Data type IDs +;; **Negative ids reserved for user-defined types** + +(def ^:const id-reserved (int 0)) ;; 1 (def ^:const id-bytes (int 2)) (def ^:const id-nil (int 3)) @@ -67,7 +70,7 @@ ;;;; Freezing (defprotocol Freezable (freeze-to-stream* [this stream])) -(defmacro ^:private write-id [s id] `(.writeByte ~s ~id)) +(defmacro write-id [s id] `(.writeByte ~s ~id)) (defmacro ^:private write-bytes [s ba] `(let [s# ~s ba# ~ba] (let [size# (alength ba#)] @@ -78,18 +81,32 @@ (defmacro ^:private write-utf8 [s x] `(write-bytes ~s (.getBytes ~x "UTF-8"))) (defmacro ^:private freeze-to-stream "Like `freeze-to-stream*` but with metadata support." - [x s] + [s x] `(let [x# ~x s# ~s] - (if-let [m# (meta x#)] - (do (write-id s# ~id-meta) - (freeze-to-stream* m# s#))) - (freeze-to-stream* x# s#))) + (when-let [m# (meta x#)] + (write-id s# ~id-meta) + (freeze-to-stream* m# s#)) + (try (freeze-to-stream* x# s#) + (catch java.lang.IllegalArgumentException _# + ;; Use Clojure reader as final fallback (after custom extensions) + (write-id s# id-reader) + (write-bytes s# (.getBytes (pr-str x#) "UTF-8")))))) + +(defn freeze-to-stream! + "Low-level API. Serializes arg (any Clojure data type) to a DataOutputStream." + [^DataOutputStream data-output-stream x & [{:keys [print-dup?] + :or {print-dup? true}}]] + (if (identical? *print-dup* print-dup?) + (freeze-to-stream data-output-stream x) + (binding [*print-dup* print-dup?] ; Expensive + (freeze-to-stream data-output-stream x)))) + (defmacro ^:private freezer "Helper to extend Freezable protocol." [type id & body] `(extend-type ~type - ~'Freezable + Freezable (~'freeze-to-stream* [~'x ~(with-meta 's {:tag 'DataOutputStream})] (write-id ~'s ~id) ~@body))) @@ -99,7 +116,7 @@ [type id & body] `(freezer ~type ~id (.writeInt ~'s (count ~'x)) - (doseq [i# ~'x] (freeze-to-stream i# ~'s)))) + (doseq [i# ~'x] (freeze-to-stream ~'s i#)))) (defmacro ^:private kv-freezer "Extends Freezable to key-value collection types." @@ -107,8 +124,8 @@ `(freezer ~type ~id (.writeInt ~'s (* 2 (count ~'x))) (doseq [[k# v#] ~'x] - (freeze-to-stream k# ~'s) - (freeze-to-stream v# ~'s)))) + (freeze-to-stream ~'s k#) + (freeze-to-stream ~'s v#)))) (freezer (Class/forName "[B") id-bytes (write-bytes s ^bytes x)) (freezer nil id-nil) @@ -147,9 +164,6 @@ (write-biginteger s (.numerator x)) (write-biginteger s (.denominator x))) -;; Use Clojure's own reader as final fallback -(freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8"))) - (def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta)) (defn- wrap-header [data-ba metadata] @@ -165,7 +179,8 @@ (defn freeze "Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to - true to produce bytes readble by Nippy < 2.x." + true to produce bytes readble by Nippy < 2.x. For custom types extend the + Clojure reader or see `extend-freeze`." ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode] :or {print-dup? true compressor snappy-compressor @@ -173,7 +188,7 @@ (when legacy-mode (assert-legacy-args compressor password)) (let [ba (ByteArrayOutputStream.) stream (DataOutputStream. ba)] - (binding [*print-dup* print-dup?] (freeze-to-stream x stream)) + (freeze-to-stream! stream x {:print-dup? print-dup?}) (let [ba (.toByteArray ba) ba (if compressor (compression/compress compressor ba) ba) ba (if password (encryption/encrypt encryptor password ba) ba)] @@ -203,13 +218,14 @@ [s coll] `(let [s# ~s] (utils/repeatedly-into ~coll (/ (.readInt s#) 2) - [(thaw-from-stream s#) (thaw-from-stream s#)]))) + [(thaw-from-stream s#) (thaw-from-stream s#)]))) + +(declare ^:private custom-readers) (defn- thaw-from-stream [^DataInputStream s] (let [type-id (.readByte s)] - (utils/case-eval - type-id + (utils/case-eval type-id id-reader (read-string (read-utf8 s)) id-bytes (read-bytes s) @@ -252,7 +268,26 @@ (* 2 (.readInt s)) (thaw-from-stream s))) id-old-keyword (keyword (.readUTF s)) - (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) + (if-not (neg? type-id) + (throw (Exception. (str "Unknown type ID: " type-id))) + + ;; Custom types + (if-let [reader (get @custom-readers type-id)] + (try (reader s) + (catch Exception e + (throw (Exception. (str "Reader exception for custom type ID: " + (- type-id)) e)))) + (throw (Exception. (str "No reader provided for custom type ID: " + (- type-id))))))))) + +(defn thaw-from-stream! + "Low-level API. Deserializes a frozen object from given DataInputStream to its + original Clojure data type." + [data-input-stream & [{:keys [read-eval?]}]] + (if (identical? *read-eval* read-eval?) + (thaw-from-stream data-input-stream) + (binding [*read-eval* read-eval?] ; Expensive + (thaw-from-stream data-input-stream)))) (defn- try-parse-header [ba] (when-let [[head-ba data-ba] (utils/ba-split ba 4)] @@ -261,12 +296,13 @@ [data-ba (head-meta meta-id {:unrecognized-header? true})])))) (defn thaw - "Deserializes frozen bytes to their original Clojure data type. Supports data - frozen with current and all previous versions of Nippy. + "Deserializes a frozen object from given byte array to its original Clojure + data type. Supports data frozen with current and all previous versions of + Nippy. For custom types extend the Clojure reader or see `extend-thaw`. WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless you are sure you know what you're doing." - [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-opts] + [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-opts readers] :or {legacy-opts {:compressed? true} compressor snappy-compressor encryptor aes128-encryptor} @@ -284,7 +320,9 @@ ba (if password (encryption/decrypt encryptor password ba) ba) ba (if compressor (compression/decompress compressor ba) ba) stream (DataInputStream. (ByteArrayInputStream. ba))] - (binding [*read-eval* read-eval?] (thaw-from-stream stream))) + + (thaw-from-stream! stream {:read-eval? read-eval?})) + (catch Exception e (cond password (ex "Wrong password/encryptor?" e) @@ -307,7 +345,9 @@ :else (try (try-thaw-data data-ba head-meta) (catch Exception e (if legacy-opts - (try-thaw-data ba nil) + (try (try-thaw-data ba nil) + (catch Exception _ + (throw e))) (throw e))))) ;; Header definitely not okay @@ -320,6 +360,39 @@ (thaw (freeze "hello" {:password [:salted "p"]})) ; ex (thaw (freeze "hello") {:password [:salted "p"]})) +;;;; Custom types + +(defmacro extend-freeze + "Alpha - subject to change. + Extends Nippy to support freezing of a custom type with id ∈[1, 128]: + (defrecord MyType [data]) + (extend-freeze MyType 1 [x data-output-stream] + (.writeUTF [data-output-stream] (:data x)))" + [type custom-type-id [x stream] & body] + (assert (and (>= custom-type-id 1) (<= custom-type-id 128))) + `(extend-type ~type + Freezable + (~'freeze-to-stream* [~x ~(with-meta stream {:tag 'java.io.DataOutputStream})] + (write-id ~stream ~(int (- custom-type-id))) + ~@body))) + +(defonce custom-readers (atom {})) ; { (fn [data-input-stream]) ...} +(defmacro extend-thaw + "Alpha - subject to change. + Extends Nippy to support thawing of a custom type with id ∈[1, 128]: + (extend-thaw 1 [data-input-stream] + (->MyType (.readUTF data-input-stream)))" + [custom-type-id [stream] & body] + (assert (and (>= custom-type-id 1) (<= custom-type-id 128))) + `(swap! custom-readers assoc ~(int (- custom-type-id)) + (fn [~(with-meta stream {:tag 'java.io.DataInputStream})] + ~@body))) + +(comment (defrecord MyType [data]) + (extend-freeze MyType 1 [x s] (.writeUTF s (:data x))) + (extend-thaw 1 [s] (->MyType (.readUTF s))) + (thaw (freeze (->MyType "Joe")))) + ;;;; Stress data (def stress-data "Reference data used for tests & benchmarks." @@ -390,4 +463,4 @@ :or {compressed? true}}] (thaw ba {:legacy-opts {:compressed? compressed?} :read-eval? read-eval? - :password nil})) + :password nil})) \ No newline at end of file diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 4889076..0cc34c8 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -30,4 +30,11 @@ (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) +;;; Custom types +(defrecord MyType [data]) +(nippy/extend-freeze MyType 1 [x s] (.writeUTF s (:data x))) +(expect Exception (thaw (freeze (->MyType "Joe")))) +(expect (MyType. "Joe") (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s))) + (thaw (freeze (->MyType "Joe"))))) + (expect (benchmarks/bench {:reader? false})) ; Also tests :cached passwords \ No newline at end of file