From a8b1686a5487562d3a47dd7323aa9d3d388414d2 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 25 Jul 2013 15:41:13 +0700 Subject: [PATCH 1/8] Make `head-meta` ^:const --- src/taoensso/nippy.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index b742f6c..be40222 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -15,7 +15,7 @@ ;;;; Nippy 2.x+ header spec (4 bytes) (def ^:private ^:const head-version 1) (def ^:private head-sig (.getBytes "NPY" "UTF-8")) -(def ^:private head-meta "Final byte stores version-dependent metadata." +(def ^:private ^:const head-meta "Final byte stores version-dependent metadata." {(byte 0) {:version 1 :compressed? false :encrypted? false} (byte 1) {:version 1 :compressed? true :encrypted? false} (byte 2) {:version 1 :compressed? false :encrypted? true} From 69611657c7a49fd597eba81a3f12bb4f81ae04a6 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 25 Jul 2013 15:51:45 +0700 Subject: [PATCH 2/8] Expose low-level fns: `freeze-to-stream!`, `thaw-from-stream!` --- src/taoensso/nippy.clj | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index be40222..5e2abab 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -78,13 +78,20 @@ (defmacro ^:private write-utf8 [s x] `(write-bytes ~s (.getBytes ~x "UTF-8"))) (defmacro ^:private freeze-to-stream "Like `freeze-to-stream*` but with metadata support." - [x s] + [s x] `(let [x# ~x s# ~s] - (if-let [m# (meta x#)] - (do (write-id s# ~id-meta) - (freeze-to-stream* m# s#))) + (when-let [m# (meta x#)] + (write-id s# ~id-meta) + (freeze-to-stream* m# s#)) (freeze-to-stream* x# s#))) +(defn freeze-to-stream! + "Low-level API. Serializes arg (any Clojure data type) to a DataOutputStream." + [^DataOutputStream data-output-stream x & [{:keys [print-dup?] + :or {print-dup? true}}]] + (binding [*print-dup* print-dup?] + (freeze-to-stream data-output-stream x))) + (defmacro ^:private freezer "Helper to extend Freezable protocol." [type id & body] @@ -99,7 +106,7 @@ [type id & body] `(freezer ~type ~id (.writeInt ~'s (count ~'x)) - (doseq [i# ~'x] (freeze-to-stream i# ~'s)))) + (doseq [i# ~'x] (freeze-to-stream ~'s i#)))) (defmacro ^:private kv-freezer "Extends Freezable to key-value collection types." @@ -107,8 +114,8 @@ `(freezer ~type ~id (.writeInt ~'s (* 2 (count ~'x))) (doseq [[k# v#] ~'x] - (freeze-to-stream k# ~'s) - (freeze-to-stream v# ~'s)))) + (freeze-to-stream ~'s k#) + (freeze-to-stream ~'s v#)))) (freezer (Class/forName "[B") id-bytes (write-bytes s ^bytes x)) (freezer nil id-nil) @@ -173,7 +180,7 @@ (when legacy-mode (assert-legacy-args compressor password)) (let [ba (ByteArrayOutputStream.) stream (DataOutputStream. ba)] - (binding [*print-dup* print-dup?] (freeze-to-stream x stream)) + (freeze-to-stream! stream x {:print-dup? print-dup?}) (let [ba (.toByteArray ba) ba (if compressor (compression/compress compressor ba) ba) ba (if password (encryption/encrypt encryptor password ba) ba)] @@ -254,6 +261,13 @@ (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) +(defn thaw-from-stream! + "Low-level API. Deserializes a frozen object from given DataInputStream to its + original Clojure data type." + [data-input-stream & [{:keys [read-eval?]}]] + (binding [*read-eval* read-eval?] + (thaw-from-stream data-input-stream))) + (defn- try-parse-header [ba] (when-let [[head-ba data-ba] (utils/ba-split ba 4)] (let [[head-sig* [meta-id]] (utils/ba-split head-ba 3)] @@ -261,8 +275,9 @@ [data-ba (head-meta meta-id {:unrecognized-header? true})])))) (defn thaw - "Deserializes frozen bytes to their original Clojure data type. Supports data - frozen with current and all previous versions of Nippy. + "Deserializes a frozen object from given byte array to its original Clojure + data type. Supports data frozen with current and all previous versions of + Nippy. WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless you are sure you know what you're doing." @@ -284,7 +299,7 @@ ba (if password (encryption/decrypt encryptor password ba) ba) ba (if compressor (compression/decompress compressor ba) ba) stream (DataInputStream. (ByteArrayInputStream. ba))] - (binding [*read-eval* read-eval?] (thaw-from-stream stream))) + (thaw-from-stream! stream {:read-eval? read-eval?})) (catch Exception e (cond password (ex "Wrong password/encryptor?" e) From 8e4cc072e35bf1590d536fcaf47e51ce1663f2c0 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 29 Jul 2013 15:11:20 +0700 Subject: [PATCH 3/8] Move Reader fallback out of Freezable protocol to make protocol extensible --- src/taoensso/nippy.clj | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 5e2abab..2e5d3a6 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -83,7 +83,11 @@ (when-let [m# (meta x#)] (write-id s# ~id-meta) (freeze-to-stream* m# s#)) - (freeze-to-stream* x# s#))) + (try (freeze-to-stream* x# s#) + (catch java.lang.IllegalArgumentException _# + ;; Use Clojure reader as final fallback (after custom extensions) + (write-id s# id-reader) + (write-bytes s# (.getBytes (pr-str x#) "UTF-8")))))) (defn freeze-to-stream! "Low-level API. Serializes arg (any Clojure data type) to a DataOutputStream." @@ -154,9 +158,6 @@ (write-biginteger s (.numerator x)) (write-biginteger s (.denominator x))) -;; Use Clojure's own reader as final fallback -(freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8"))) - (def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta)) (defn- wrap-header [data-ba metadata] From 99091b0a32f02a613a5d0383bcf216383ad13a89 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 29 Jul 2013 15:22:31 +0700 Subject: [PATCH 4/8] Add support for custom thaw readers --- src/taoensso/nippy.clj | 48 ++++++++++++++++++++++-------- test/taoensso/nippy/tests/main.clj | 11 +++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 2e5d3a6..001991f 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -23,6 +23,9 @@ ;;;; Data type IDs +;; **Negative ids reserved for user-defined types** + +(def ^:const id-reserved (int 0)) ;; 1 (def ^:const id-bytes (int 2)) (def ^:const id-nil (int 3)) @@ -65,7 +68,7 @@ (def ^:const id-old-keyword (int 12)) ; as of 2.0.0-alpha5, for str consistecy ;;;; Freezing -(defprotocol Freezable (freeze-to-stream* [this stream])) +(defprotocol Freezable (freeze-to-stream* [this ^DataOutputStream stream])) (defmacro ^:private write-id [s id] `(.writeByte ~s ~id)) (defmacro ^:private write-bytes [s ba] @@ -173,7 +176,14 @@ (defn freeze "Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to - true to produce bytes readble by Nippy < 2.x." + true to produce bytes readble by Nippy < 2.x. + + For custom types extend the Clojure reader or Nippy's `Freezable` protocol: + (defrecord MyType [data] + nippy/Freezable + (freeze-to-stream* [x stream] + (.writeByte stream -1) ; Custom type id ∈ [-128, -1] + (.writeUTF stream (:data x))))" ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode] :or {print-dup? true compressor snappy-compressor @@ -214,10 +224,9 @@ [(thaw-from-stream s#) (thaw-from-stream s#)]))) (defn- thaw-from-stream - [^DataInputStream s] + [^DataInputStream s & [readers]] (let [type-id (.readByte s)] - (utils/case-eval - type-id + (utils/case-eval type-id id-reader (read-string (read-utf8 s)) id-bytes (read-bytes s) @@ -260,14 +269,22 @@ (* 2 (.readInt s)) (thaw-from-stream s))) id-old-keyword (keyword (.readUTF s)) - (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) + ;;; Custom types + (or (when-let [reader (get readers type-id)] + (try (reader s) + (catch Exception e + (throw (Exception. (str "Reader exception for custom type ID: " + type-id) e))))) + (if (neg? type-id) + (throw (Exception. (str "No reader provided for custom type ID: " type-id))) + (throw (Exception. (str "Unknown type ID: " type-id)))))))) (defn thaw-from-stream! "Low-level API. Deserializes a frozen object from given DataInputStream to its original Clojure data type." - [data-input-stream & [{:keys [read-eval?]}]] + [data-input-stream & [{:keys [read-eval? readers]}]] (binding [*read-eval* read-eval?] - (thaw-from-stream data-input-stream))) + (thaw-from-stream data-input-stream readers))) (defn- try-parse-header [ba] (when-let [[head-ba data-ba] (utils/ba-split ba 4)] @@ -280,9 +297,12 @@ data type. Supports data frozen with current and all previous versions of Nippy. + For custom `Freezable` types provide a `:readers` arg: + (thaw (freeze (MyType. \"Joe\")) {:readers {-1 (fn [stream] (.readUTF stream))}}) + WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless you are sure you know what you're doing." - [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-opts] + [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-opts readers] :or {legacy-opts {:compressed? true} compressor snappy-compressor encryptor aes128-encryptor} @@ -300,7 +320,9 @@ ba (if password (encryption/decrypt encryptor password ba) ba) ba (if compressor (compression/decompress compressor ba) ba) stream (DataInputStream. (ByteArrayInputStream. ba))] - (thaw-from-stream! stream {:read-eval? read-eval?})) + + (thaw-from-stream! stream {:read-eval? read-eval? :readers readers})) + (catch Exception e (cond password (ex "Wrong password/encryptor?" e) @@ -323,7 +345,9 @@ :else (try (try-thaw-data data-ba head-meta) (catch Exception e (if legacy-opts - (try-thaw-data ba nil) + (try (try-thaw-data ba nil) + (catch Exception _ + (throw e))) (throw e))))) ;; Header definitely not okay @@ -406,4 +430,4 @@ :or {compressed? true}}] (thaw ba {:legacy-opts {:compressed? compressed?} :read-eval? read-eval? - :password nil})) + :password nil})) \ No newline at end of file diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 4889076..794e9cb 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -30,4 +30,15 @@ (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) +;;; Custom types +(defrecord MyType [data] + nippy/Freezable + (freeze-to-stream* [x s] + (.writeByte s -1) + (.writeUTF s (:data x)))) + +(expect Exception (thaw (freeze (MyType. "Joe")))) +(expect "Joe" (thaw (freeze (MyType. "Joe")) + {:readers {-1 (fn [s] (.readUTF s))}})) + (expect (benchmarks/bench {:reader? false})) ; Also tests :cached passwords \ No newline at end of file From c2a964932cda5e335e0582e73bac464b6871b24c Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Fri, 2 Aug 2013 14:41:59 +0700 Subject: [PATCH 5/8] Add `custom-freezer` macro for easier Freezable extension --- src/taoensso/nippy.clj | 44 +++++++++++++++++++----------- test/taoensso/nippy/tests/main.clj | 12 ++++---- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 001991f..ce7c0cc 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -68,9 +68,9 @@ (def ^:const id-old-keyword (int 12)) ; as of 2.0.0-alpha5, for str consistecy ;;;; Freezing -(defprotocol Freezable (freeze-to-stream* [this ^DataOutputStream stream])) +(defprotocol Freezable (freeze-to-stream* [this stream])) -(defmacro ^:private write-id [s id] `(.writeByte ~s ~id)) +(defmacro write-id [s id] `(.writeByte ~s ~id)) (defmacro ^:private write-bytes [s ba] `(let [s# ~s ba# ~ba] (let [size# (alength ba#)] @@ -84,7 +84,7 @@ [s x] `(let [x# ~x s# ~s] (when-let [m# (meta x#)] - (write-id s# ~id-meta) + (write-id s# ~id-meta) (freeze-to-stream* m# s#)) (try (freeze-to-stream* x# s#) (catch java.lang.IllegalArgumentException _# @@ -103,11 +103,27 @@ "Helper to extend Freezable protocol." [type id & body] `(extend-type ~type - ~'Freezable + Freezable (~'freeze-to-stream* [~'x ~(with-meta 's {:tag 'DataOutputStream})] (write-id ~'s ~id) ~@body))) +(defmacro custom-freezer + "Helper to extend Freezable protocol to custom types with id ∈[1, 128]: + (defrecord MyType [data]) + (custom-freezer MyType 1 x s (.writeUTF s (:data x)))" + [type id x data-output-stream & body] + (assert (and (>= id 1) (<= id 128))) + `(extend-type ~type + Freezable + (~'freeze-to-stream* [~x ~(with-meta data-output-stream + {:tag 'DataOutputStream})] + (write-id ~data-output-stream ~(int (- id))) + ~@body))) + +(comment (defrecord MyType [data]) + (custom-freezer MyType 1 x s (.writeUTF s (:data x)))) + (defmacro ^:private coll-freezer "Extends Freezable to simple collection types." [type id & body] @@ -176,14 +192,8 @@ (defn freeze "Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to - true to produce bytes readble by Nippy < 2.x. - - For custom types extend the Clojure reader or Nippy's `Freezable` protocol: - (defrecord MyType [data] - nippy/Freezable - (freeze-to-stream* [x stream] - (.writeByte stream -1) ; Custom type id ∈ [-128, -1] - (.writeUTF stream (:data x))))" + true to produce bytes readble by Nippy < 2.x. For custom types extend the + Clojure reader or see `custom-freezer`." ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode] :or {print-dup? true compressor snappy-compressor @@ -270,13 +280,14 @@ id-old-keyword (keyword (.readUTF s)) ;;; Custom types - (or (when-let [reader (get readers type-id)] + (or (when-let [reader (get readers (- type-id))] (try (reader s) (catch Exception e (throw (Exception. (str "Reader exception for custom type ID: " - type-id) e))))) + (- type-id)) e))))) (if (neg? type-id) - (throw (Exception. (str "No reader provided for custom type ID: " type-id))) + (throw (Exception. (str "No reader provided for custom type ID: " + (- type-id)))) (throw (Exception. (str "Unknown type ID: " type-id)))))))) (defn thaw-from-stream! @@ -298,7 +309,8 @@ Nippy. For custom `Freezable` types provide a `:readers` arg: - (thaw (freeze (MyType. \"Joe\")) {:readers {-1 (fn [stream] (.readUTF stream))}}) + (thaw (freeze (MyType. \"Joe\")) + {:readers {1 (fn [^DataInputStream stream] (.readUTF stream))}}) WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless you are sure you know what you're doing." diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 794e9cb..6acfce8 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -1,7 +1,8 @@ (ns taoensso.nippy.tests.main (:require [expectations :as test :refer :all] [taoensso.nippy :as nippy :refer (freeze thaw)] - [taoensso.nippy.benchmarks :as benchmarks])) + [taoensso.nippy.benchmarks :as benchmarks]) + (:import [java.io DataInputStream DataOutputStream])) ;; Remove stuff from stress-data that breaks roundtrip equality (def test-data (dissoc nippy/stress-data :bytes)) @@ -31,14 +32,11 @@ (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) ;;; Custom types -(defrecord MyType [data] - nippy/Freezable - (freeze-to-stream* [x s] - (.writeByte s -1) - (.writeUTF s (:data x)))) +(defrecord MyType [data]) +(nippy/custom-freezer MyType 1 x s (.writeUTF s (:data x))) (expect Exception (thaw (freeze (MyType. "Joe")))) (expect "Joe" (thaw (freeze (MyType. "Joe")) - {:readers {-1 (fn [s] (.readUTF s))}})) + {:readers {1 (fn [^DataInputStream s] (.readUTF s))}})) (expect (benchmarks/bench {:reader? false})) ; Also tests :cached passwords \ No newline at end of file From 4071d0f3ec73489c6f36bf8266f19f2ca4f8e5aa Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Fri, 2 Aug 2013 15:20:14 +0700 Subject: [PATCH 6/8] Switch to simpler `extend-freeze`, `extend-thaw` custom type API (ALPHA) --- src/taoensso/nippy.clj | 90 ++++++++++++++++++------------ test/taoensso/nippy/tests/main.clj | 12 ++-- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index ce7c0cc..bab6a48 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -108,22 +108,6 @@ (write-id ~'s ~id) ~@body))) -(defmacro custom-freezer - "Helper to extend Freezable protocol to custom types with id ∈[1, 128]: - (defrecord MyType [data]) - (custom-freezer MyType 1 x s (.writeUTF s (:data x)))" - [type id x data-output-stream & body] - (assert (and (>= id 1) (<= id 128))) - `(extend-type ~type - Freezable - (~'freeze-to-stream* [~x ~(with-meta data-output-stream - {:tag 'DataOutputStream})] - (write-id ~data-output-stream ~(int (- id))) - ~@body))) - -(comment (defrecord MyType [data]) - (custom-freezer MyType 1 x s (.writeUTF s (:data x)))) - (defmacro ^:private coll-freezer "Extends Freezable to simple collection types." [type id & body] @@ -193,7 +177,7 @@ (defn freeze "Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to true to produce bytes readble by Nippy < 2.x. For custom types extend the - Clojure reader or see `custom-freezer`." + Clojure reader or see `extend-freeze`." ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode] :or {print-dup? true compressor snappy-compressor @@ -231,10 +215,12 @@ [s coll] `(let [s# ~s] (utils/repeatedly-into ~coll (/ (.readInt s#) 2) - [(thaw-from-stream s#) (thaw-from-stream s#)]))) + [(thaw-from-stream s#) (thaw-from-stream s#)]))) + +(declare ^:private custom-readers) (defn- thaw-from-stream - [^DataInputStream s & [readers]] + [^DataInputStream s] (let [type-id (.readByte s)] (utils/case-eval type-id @@ -279,23 +265,24 @@ (* 2 (.readInt s)) (thaw-from-stream s))) id-old-keyword (keyword (.readUTF s)) - ;;; Custom types - (or (when-let [reader (get readers (- type-id))] - (try (reader s) - (catch Exception e - (throw (Exception. (str "Reader exception for custom type ID: " - (- type-id)) e))))) - (if (neg? type-id) - (throw (Exception. (str "No reader provided for custom type ID: " - (- type-id)))) - (throw (Exception. (str "Unknown type ID: " type-id)))))))) + (if-not (neg? type-id) + (throw (Exception. (str "Unknown type ID: " type-id))) + + ;; Custom types + (if-let [reader (get @custom-readers type-id)] + (try (reader s) + (catch Exception e + (throw (Exception. (str "Reader exception for custom type ID: " + (- type-id)) e)))) + (throw (Exception. (str "No reader provided for custom type ID: " + (- type-id))))))))) (defn thaw-from-stream! "Low-level API. Deserializes a frozen object from given DataInputStream to its original Clojure data type." - [data-input-stream & [{:keys [read-eval? readers]}]] + [data-input-stream & [{:keys [read-eval?]}]] (binding [*read-eval* read-eval?] - (thaw-from-stream data-input-stream readers))) + (thaw-from-stream data-input-stream))) (defn- try-parse-header [ba] (when-let [[head-ba data-ba] (utils/ba-split ba 4)] @@ -306,11 +293,7 @@ (defn thaw "Deserializes a frozen object from given byte array to its original Clojure data type. Supports data frozen with current and all previous versions of - Nippy. - - For custom `Freezable` types provide a `:readers` arg: - (thaw (freeze (MyType. \"Joe\")) - {:readers {1 (fn [^DataInputStream stream] (.readUTF stream))}}) + Nippy. For custom types extend the Clojure reader or see `extend-thaw`. WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless you are sure you know what you're doing." @@ -333,7 +316,7 @@ ba (if compressor (compression/decompress compressor ba) ba) stream (DataInputStream. (ByteArrayInputStream. ba))] - (thaw-from-stream! stream {:read-eval? read-eval? :readers readers})) + (thaw-from-stream! stream {:read-eval? read-eval?})) (catch Exception e (cond @@ -372,6 +355,39 @@ (thaw (freeze "hello" {:password [:salted "p"]})) ; ex (thaw (freeze "hello") {:password [:salted "p"]})) +;;;; Custom types + +(defmacro extend-freeze + "Alpha - subject to change. + Extends Nippy to support freezing of a custom type with id ∈[1, 128]: + (defrecord MyType [data]) + (extend-freeze MyType 1 [x data-output-stream] + (.writeUTF [data-output-stream] (:data x)))" + [type custom-type-id [x stream] & body] + (assert (and (>= custom-type-id 1) (<= custom-type-id 128))) + `(extend-type ~type + Freezable + (~'freeze-to-stream* [~x ~(with-meta stream {:tag 'java.io.DataOutputStream})] + (write-id ~stream ~(int (- custom-type-id))) + ~@body))) + +(defonce custom-readers (atom {})) ; { (fn [data-input-stream]) ...} +(defmacro extend-thaw + "Alpha - subject to change. + Extends Nippy to support thawing of a custom type with id ∈[1, 128]: + (extend-thaw 1 [data-input-stream] + (->MyType (.readUTF data-input-stream)))" + [custom-type-id [stream] & body] + (assert (and (>= custom-type-id 1) (<= custom-type-id 128))) + `(swap! custom-readers assoc ~(int (- custom-type-id)) + (fn [~(with-meta stream {:tag 'java.io.DataInputStream})] + ~@body))) + +(comment (defrecord MyType [data]) + (extend-freeze MyType 1 [x s] (.writeUTF s (:data x))) + (extend-thaw 1 [s] (->MyType (.readUTF s))) + (thaw (freeze (->MyType "Joe")))) + ;;;; Stress data (def stress-data "Reference data used for tests & benchmarks." diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 6acfce8..0cc34c8 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -1,8 +1,7 @@ (ns taoensso.nippy.tests.main (:require [expectations :as test :refer :all] [taoensso.nippy :as nippy :refer (freeze thaw)] - [taoensso.nippy.benchmarks :as benchmarks]) - (:import [java.io DataInputStream DataOutputStream])) + [taoensso.nippy.benchmarks :as benchmarks])) ;; Remove stuff from stress-data that breaks roundtrip equality (def test-data (dissoc nippy/stress-data :bytes)) @@ -33,10 +32,9 @@ ;;; Custom types (defrecord MyType [data]) -(nippy/custom-freezer MyType 1 x s (.writeUTF s (:data x))) - -(expect Exception (thaw (freeze (MyType. "Joe")))) -(expect "Joe" (thaw (freeze (MyType. "Joe")) - {:readers {1 (fn [^DataInputStream s] (.readUTF s))}})) +(nippy/extend-freeze MyType 1 [x s] (.writeUTF s (:data x))) +(expect Exception (thaw (freeze (->MyType "Joe")))) +(expect (MyType. "Joe") (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s))) + (thaw (freeze (->MyType "Joe"))))) (expect (benchmarks/bench {:reader? false})) ; Also tests :cached passwords \ No newline at end of file From c69bb0ec5fcbf7b2b33ac59c43d8c08fbe07be59 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Fri, 2 Aug 2013 16:28:05 +0700 Subject: [PATCH 7/8] Perf: only set bindings when necessary --- src/taoensso/nippy.clj | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index bab6a48..9a6a541 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -96,8 +96,11 @@ "Low-level API. Serializes arg (any Clojure data type) to a DataOutputStream." [^DataOutputStream data-output-stream x & [{:keys [print-dup?] :or {print-dup? true}}]] - (binding [*print-dup* print-dup?] - (freeze-to-stream data-output-stream x))) + (if (identical? *print-dup* print-dup?) + (freeze-to-stream data-output-stream x) + (binding [*print-dup* print-dup?] ; Expensive + (freeze-to-stream data-output-stream x)))) + (defmacro ^:private freezer "Helper to extend Freezable protocol." @@ -281,8 +284,10 @@ "Low-level API. Deserializes a frozen object from given DataInputStream to its original Clojure data type." [data-input-stream & [{:keys [read-eval?]}]] - (binding [*read-eval* read-eval?] - (thaw-from-stream data-input-stream))) + (if (identical? *read-eval* read-eval?) + (thaw-from-stream data-input-stream) + (binding [*read-eval* read-eval?] ; Expensive + (thaw-from-stream data-input-stream)))) (defn- try-parse-header [ba] (when-let [[head-ba data-ba] (utils/ba-split ba 4)] From bce0ea45a5c0c2ad8c9022fa1cdcc7e5555157ea Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 29 Jul 2013 15:59:24 +0700 Subject: [PATCH 8/8] v2.1.0 --- CHANGELOG.md | 11 +++++++++++ README.md | 25 +++++++++++++++++++++---- project.clj | 2 +- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bcf319..4167ef8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v2.0.0 → v2.1.0 + * Exposed low-level fns: `freeze-to-stream!`, `thaw-from-stream!`. + * Added `extend-freeze` and `extend-thaw` for extending to custom types: + * Added support for easily extending Nippy de/serialization to custom types: + ```clojure + (defrecord MyType [data]) + (nippy/extend-freeze MyType 1 [x steam] (.writeUTF stream (:data x))) + (nippy/extend-thaw 1 [stream] (->MyType (.readUTF stream))) + (nippy/thaw (nippy/freeze (->MyType "Joe"))) => #taoensso.nippy.MyType{:data "Joe"} + ``` + ## v1.2.1 → v2.0.0 * **MIGRATION NOTE**: Please be sure to use `lein clean` to clear old (v1) build artifacts! * Refactored for huge performance improvements (~40% roundtrip time). diff --git a/README.md b/README.md index 5994fdc..1b253c6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ **[API docs](http://ptaoussanis.github.io/nippy/)** | **[CHANGELOG](https://github.com/ptaoussanis/nippy/blob/master/CHANGELOG.md)** | [contact & contributing](#contact--contributing) | [other Clojure libs](https://www.taoensso.com/clojure-libraries) | [Twitter](https://twitter.com/#!/ptaoussanis) | current [semantic](http://semver.org/) version: ```clojure -[com.taoensso/nippy "2.0.0"] ; See CHANGELOG for changes since 1.x +[com.taoensso/nippy "2.1.0"] ; See CHANGELOG for changes since 1.x ``` v2 adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), easier integration into other tools/libraries, and hugely improved performance. @@ -20,8 +20,9 @@ Nippy is an attempt to provide a reliable, high-performance **drop-in alternativ ## What's in the box™? * Small, uncomplicated **all-Clojure** library. * **Great performance**. - * Comprehesive, extensible **support for all major data types**. - * **Reader-fallback** for difficult/future types (including Clojure 1.4+ tagged literals). + * Comprehesive **support for all standard data types**. + * **Easily extendable to custom data types**. (v2.1+) + * **Reader-fallback** for all other types (including Clojure 1.4+ tagged literals). * **Full test coverage** for every supported type. * Fully pluggable **compression**, including built-in high-performance [Snappy](http://code.google.com/p/snappy/) compressor. * Fully pluggable **encryption**, including built-in high-strength AES128 enabled with a single `:password [:salted "my-password"]` option. (v2+) @@ -34,7 +35,7 @@ Nippy is an attempt to provide a reliable, high-performance **drop-in alternativ Add the necessary dependency to your [Leiningen](http://leiningen.org/) `project.clj` and `require` the library in your ns: ```clojure -[com.taoensso/nippy "2.0.0"] ; project.clj +[com.taoensso/nippy "2.1.0"] ; project.clj (ns my-app (:require [taoensso.nippy :as nippy])) ; ns ``` @@ -117,6 +118,22 @@ Nippy v2+ also gives you **dead simple data encryption**. Add a single option to There's two default forms of encryption on offer: `:salted` and `:cached`. Each of these makes carefully-chosen trade-offs and is suited to one of two common use cases. See the `aes128-encryptor` [docstring](http://ptaoussanis.github.io/nippy/taoensso.nippy.encryption.html) for a detailed explanation of why/when you'd want one or the other. +### Custom types (v2.1+, ALPHA - subject to change) + +```clojure +(defrecord MyType [data]) + +(nippy/extend-freeze MyType 1 ; A unique type id ∈[1, 128] + [x data-output-steam] + (.writeUTF data-output-stream (:data x))) + +(nippy/extend-thaw 1 ; Same type id + [data-input-stream] + (->MyType (.readUTF data-input-stream))) + +(nippy/thaw (nippy/freeze (->MyType "Joe"))) => #taoensso.nippy.MyType{:data "Joe"} +``` + ## Performance ![Comparison chart](https://github.com/ptaoussanis/nippy/raw/master/benchmarks.png) diff --git a/project.clj b/project.clj index 73f4592..7c22550 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.0.0" +(defproject com.taoensso/nippy "2.1.0" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License"