Merge branch 'dev' (Serializable)

CHANGELOG.md

@@ -1,3 +1,8 @@
+## v2.4.1 → v2.5.0-beta1
+  * Refactored standard Freezable protocol implementations to de-emphasise interfaces as a matter of hygiene, Ref. http://goo.gl/IFXzvh.
+  * BETA STATUS: Added an addition (pre-Reader) Serializable fallback. This should greatly extend the number of out-the-box-serializable types.
+
+
 ## v2.3.0 → v2.4.1
   * Added (alpha) LZMA2 (high-ratio) compressor.
   * Bump tools.reader dependency to 0.7.9.

README.md

@@ -2,6 +2,7 @@
 
 ```clojure
 [com.taoensso/nippy "2.4.1"] ; Stable; see CHANGELOG for changes since 1.x
+[com.taoensso/nippy "2.5.0-beta1"] ; Development; adds Serializable fallback
 ```
 
 v2 adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), easier integration into other tools/libraries, and hugely improved performance.
@@ -22,6 +23,7 @@ Nippy is an attempt to provide a reliable, high-performance **drop-in alternativ
   * **Great performance**.
   * Comprehesive **support for all standard data types**.
   * **Easily extendable to custom data types**. (v2.1+)
+  * Java's **Serializable** fallback when available. (v2.5+)
   * **Reader-fallback** for all other types (including Clojure 1.4+ tagged literals).
   * **Full test coverage** for every supported type.
   * Fully pluggable **compression**, including built-in high-performance [Snappy](http://code.google.com/p/snappy/) compressor.
@@ -87,8 +89,15 @@ nippy/stress-data
  :bigdec       (bigdec 3.1415926535897932384626433832795)
 
  :ratio        22/7
- :tagged-uuid  (java.util.UUID/randomUUID)
+ :uuid         (java.util.UUID/randomUUID)
- :tagged-date  (java.util.Date.)}
+ :date         (java.util.Date.)
+
+ :stress-record (->StressRecord "data")
+
+ ;; Serializable
+ :throwable    (Throwable. "Yolo")
+ :exception    (try (/ 1 0) (catch Exception e e))
+ :ex-info      (ex-info "ExInfo" {:data "data"})}
 ```
 
 Serialize it:

project.clj

@@ -1,4 +1,4 @@
-(defproject com.taoensso/nippy "2.4.1"
+(defproject com.taoensso/nippy "2.5.0-beta1"
   :description "Clojure serialization library"
   :url "https://github.com/ptaoussanis/nippy"
   :license {:name "Eclipse Public License"
@@ -21,7 +21,7 @@
             "codox"       ["with-profile" "+test" "doc"]}
   :plugins [[lein-expectations "0.0.8"]
             [lein-autoexpect   "1.0"]
-            [lein-ancient      "0.4.4"]
+            [lein-ancient      "0.5.1"]
             [codox             "0.6.6"]]
   :min-lein-version "2.0.0"
   :global-vars {*warn-on-reflection* true}

src/taoensso/nippy.clj

@@ -8,12 +8,16 @@
              (compression :as compression :refer (snappy-compressor))
              (encryption  :as encryption  :refer (aes128-encryptor))])
   (:import  [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
-             DataOutputStream]
+             DataOutputStream Serializable]
             [java.lang.reflect Method]
             [java.util Date UUID]
-            [clojure.lang Keyword BigInt Ratio PersistentQueue PersistentTreeMap
+            [clojure.lang Keyword BigInt Ratio
-             PersistentTreeSet IPersistentList IPersistentVector IPersistentMap
+             APersistentMap APersistentVector APersistentSet
-             APersistentMap IPersistentSet ISeq IRecord]))
+             IPersistentList IPersistentMap ; IPersistentVector IPersistentSet
+             PersistentQueue PersistentTreeMap PersistentTreeSet ; PersistentList
+             LazySeq
+             IRecord ; ISeq
+             ]))
 
 ;;;; Nippy 2.x+ header spec (4 bytes)
 (def ^:private ^:const head-version 1)
@@ -33,7 +37,8 @@
 (def ^:const id-bytes      (int 2))
 (def ^:const id-nil        (int 3))
 (def ^:const id-boolean    (int 4))
-(def ^:const id-reader     (int 5)) ; Fallback: pr-str output
+(def ^:const id-reader     (int 5)) ; Fallback #2: pr-str output
+(def ^:const id-serializable (int 6)) ; Fallback #1
 
 (def ^:const id-char       (int 10))
 ;;                              11
@@ -65,6 +70,7 @@
 (def ^:const id-ratio      (int 70))
 
 (def ^:const id-record     (int 80))
+;; (def ^:const id-type    (int 81)) ; TODO
 
 (def ^:const id-date       (int 90))
 (def ^:const id-uuid       (int 91))
@@ -76,7 +82,10 @@
 (def ^:const id-old-keyword (int 12)) ; as of 2.0.0-alpha5, for str consistecy
 
 ;;;; Freezing
-(defprotocol Freezable (freeze-to-stream* [this stream]))
+
+(defprotocol Freezable
+  "Be careful about extending to interfaces, Ref. http://goo.gl/6gGRlU."
+ (freeze-to-stream* [this stream]))
 
 (defmacro           write-id    [s id] `(.writeByte ~s ~id))
 (defmacro ^:private write-bytes [s ba]
@@ -96,43 +105,29 @@
        (freeze-to-stream* m# s#))
      (freeze-to-stream* x# s#)))
 
-(defn freeze-to-stream!
+(defmacro ^:private freezer [type id & body]
-  "Low-level API. Serializes arg (any Clojure data type) to a DataOutputStream."
-  [^DataOutputStream data-output-stream x & _]
-  (freeze-to-stream data-output-stream x))
-
-(defmacro ^:private freezer
-  "Helper to extend Freezable protocol."
-  [type id & body]
   `(extend-type ~type
      Freezable
      (~'freeze-to-stream* [~'x ~(with-meta 's {:tag 'DataOutputStream})]
        (write-id ~'s ~id)
        ~@body)))
 
-(defmacro ^:private coll-freezer
+(defmacro ^:private freezer-coll [type id & body]
-  "Extends Freezable to simple collection types."
-  [type id & body]
   `(freezer ~type ~id
      (if (counted? ~'x)
-       (do
+       (do (.writeInt ~'s (count ~'x))
-         (.writeInt ~'s (count ~'x))
+           (doseq [i# ~'x] (freeze-to-stream ~'s i#)))
-         (doseq [i# ~'x] (freeze-to-stream ~'s i#)))
        (let [bas# (ByteArrayOutputStream.)
-             s# (DataOutputStream. bas#)
+             s#   (DataOutputStream. bas#)
-             cnt# (reduce
+             cnt# (reduce (fn [cnt# i#]
-                    (fn [cnt# i#]
+                            (freeze-to-stream s# i#)
-                      (freeze-to-stream! s# i#)
+                            (unchecked-inc cnt#))
-                      (unchecked-inc cnt#))
+                          0 ~'x)
-                    0
-                    ~'x)
              ba# (.toByteArray bas#)]
          (.writeInt ~'s cnt#)
          (.write ~'s ba# 0 (alength ba#))))))
 
-(defmacro ^:private kv-freezer
+(defmacro ^:private freezer-kvs [type id & body]
-  "Extends Freezable to key-value collection types."
-  [type id & body]
   `(freezer ~type ~id
     (.writeInt ~'s (* 2 (count ~'x)))
     (doseq [kv# ~'x]
@@ -149,20 +144,20 @@
                                               (str ns "/" (name x))
                                               (name x))))
 
+(freezer-coll PersistentQueue       id-queue)
+(freezer-coll PersistentTreeSet     id-sorted-set)
+(freezer-kvs  PersistentTreeMap     id-sorted-map)
+
+(freezer-kvs  APersistentMap        id-map)
+(freezer-coll APersistentVector     id-vector)
+(freezer-coll APersistentSet        id-set)
+(freezer-coll IPersistentList       id-list) ; No APersistentList
+(freezer-coll LazySeq               id-seq)
+
 (freezer IRecord id-record
-         (write-utf8 s (.getName (class x)))
+         (write-utf8 s (.getName (class x))) ; Reflect
          (freeze-to-stream s (into {} x)))
 
-(coll-freezer PersistentQueue       id-queue)
-(coll-freezer PersistentTreeSet     id-sorted-set)
-(kv-freezer   PersistentTreeMap     id-sorted-map)
-
-(coll-freezer IPersistentList       id-list)
-(coll-freezer IPersistentVector     id-vector)
-(coll-freezer IPersistentSet        id-set)
-(kv-freezer   APersistentMap        id-map)
-(coll-freezer ISeq                  id-seq)
-
 (freezer Byte       id-byte    (.writeByte s x))
 (freezer Short      id-short   (.writeShort s x))
 (freezer Integer    id-integer (.writeInt s x))
@@ -185,8 +180,22 @@
          (.writeLong s (.getMostSignificantBits x))
          (.writeLong s (.getLeastSignificantBits x)))
 
-;; Use Clojure's own reader as final fallback
+;; Fallbacks. Note that we'll extend *only* to (lowly) Object to prevent
-(freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8")))
+;; interfering with higher-level implementations, Ref. http://goo.gl/6f7SKl
+(extend-type Object
+  Freezable
+  (freeze-to-stream* [x ^DataOutputStream s]
+    (if (instance? Serializable x)
+      (do ;; Fallback #1: Java's Serializable interface
+        ;;(println (format "DEBUG - Serializable fallback: %s" (type x)))
+        (write-id s id-serializable)
+        (write-utf8 s (.getName (class x))) ; Reflect
+        (.writeObject (java.io.ObjectOutputStream. s) x))
+
+      (do ;; Fallback #2: Clojure's Reader
+        ;;(println (format "DEBUG - Reader fallback: %s" (type x)))
+        (write-id s id-reader)
+        (write-bytes s (.getBytes (pr-str x) "UTF-8"))))))
 
 (def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta))
 
@@ -201,6 +210,11 @@
 
 (declare assert-legacy-args)
 
+(defn freeze-to-stream!
+  "Low-level API. Serializes arg (any Clojure data type) to a DataOutputStream."
+  [^DataOutputStream data-output-stream x & _]
+  (freeze-to-stream data-output-stream x))
+
 (defn freeze
   "Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to
   true to produce bytes readable by Nippy < 2.x. For custom types extend the
@@ -232,16 +246,12 @@
 (defmacro ^:private read-biginteger [s] `(BigInteger. (read-bytes ~s)))
 (defmacro ^:private read-utf8       [s] `(String. (read-bytes ~s) "UTF-8"))
 
-(defmacro ^:private coll-thaw "Thaws simple collection types."
+(defmacro ^:private read-coll [s coll]
-  [s coll]
+  `(let [s# ~s] (utils/repeatedly-into ~coll (.readInt s#) (thaw-from-stream s#))))
-  `(let [s# ~s]
-     (utils/repeatedly-into ~coll (.readInt s#) (thaw-from-stream s#))))
 
-(defmacro ^:private coll-thaw-kvs "Thaws key-value collection types."
+(defmacro ^:private read-kvs [s coll]
-  [s coll]
+  `(let [s# ~s] (utils/repeatedly-into ~coll (/ (.readInt s#) 2)
-  `(let [s# ~s]
+                  [(thaw-from-stream s#) (thaw-from-stream s#)])))
-     (utils/repeatedly-into ~coll (/ (.readInt s#) 2)
-                            [(thaw-from-stream s#) (thaw-from-stream s#)])))
 
 (declare ^:private custom-readers)
 
@@ -251,6 +261,10 @@
     (utils/case-eval type-id
 
      id-reader  (edn/read-string {:readers *data-readers*} (read-utf8 s))
+     id-serializable
+     (let [class ^Class (Class/forName (read-utf8 s))]
+       (cast class (.readObject (java.io.ObjectInputStream. s))))
+
      id-bytes   (read-bytes s)
      id-nil     nil
      id-boolean (.readBoolean s)
@@ -259,15 +273,15 @@
      id-string  (read-utf8 s)
      id-keyword (keyword (read-utf8 s))
 
-     id-queue      (coll-thaw s (PersistentQueue/EMPTY))
+     id-queue      (read-coll s (PersistentQueue/EMPTY))
-     id-sorted-set (coll-thaw s     (sorted-set))
+     id-sorted-set (read-coll s (sorted-set))
-     id-sorted-map (coll-thaw-kvs s (sorted-map))
+     id-sorted-map (read-kvs  s (sorted-map))
 
-     id-list    (into '() (rseq (coll-thaw s [])))
+     id-list    (into '() (rseq (read-coll s [])))
-     id-vector  (coll-thaw s  [])
+     id-vector  (read-coll s  [])
-     id-set     (coll-thaw s #{})
+     id-set     (read-coll s #{})
-     id-map     (coll-thaw-kvs s  {})
+     id-map     (read-kvs  s  {})
-     id-seq     (coll-thaw s [])
+     id-seq     (read-coll s  [])
 
      id-meta (let [m (thaw-from-stream s)] (with-meta (thaw-from-stream s) m))
 
@@ -390,7 +404,8 @@
 
 (defmacro extend-freeze
   "Alpha - subject to change.
-  Extends Nippy to support freezing of a custom type with id ∈[1, 128]:
+  Extends Nippy to support freezing of a custom type (ideally concrete) with
+  id ∈[1, 128]:
   (defrecord MyType [data])
   (extend-freeze MyType 1 [x data-output-stream]
     (.writeUTF [data-output-stream] (:data x)))"
@@ -421,6 +436,7 @@
 
 ;;;; Stress data
 
+(defrecord StressRecord [data])
 (def stress-data "Reference data used for tests & benchmarks."
   (let []
     {:bytes        (byte-array [(byte 1) (byte 2) (byte 3)])
@@ -462,10 +478,15 @@
      :bigdec       (bigdec 3.1415926535897932384626433832795)
 
      :ratio        22/7
+     :uuid         (java.util.UUID/randomUUID)
+     :date         (java.util.Date.)
 
-     ;; Clojure 1.4+ tagged literals
+     :stress-record (->StressRecord "data")
-     :tagged-uuid  (java.util.UUID/randomUUID)
+
-     :tagged-date  (java.util.Date.)}))
+     ;; Serializable
+     :throwable    (Throwable. "Yolo")
+     :exception    (try (/ 1 0) (catch Exception e e))
+     :ex-info      (ex-info "ExInfo" {:data "data"})}))
 
 ;;;; Deprecated API
 

test/taoensso/nippy/tests/main.clj

@@ -5,7 +5,7 @@
             [taoensso.nippy.benchmarks  :as benchmarks]))
 
 ;; Remove stuff from stress-data that breaks roundtrip equality
-(def test-data (dissoc nippy/stress-data :bytes))
+(def test-data (dissoc nippy/stress-data :bytes :throwable :exception :ex-info))
 
 (defn- before-run {:expectations-options :before-run} [])
 (defn- after-run  {:expectations-options :after-run}  [])
@@ -39,18 +39,16 @@
       (thaw (org.iq80.snappy.Snappy/uncompress   iq80-ba    0 (alength iq80-ba)))
       (thaw (org.iq80.snappy.Snappy/uncompress   xerial-ba  0 (alength xerial-ba))))))
 
-;;; Records (reflecting)
-(defrecord MyRec [data])
-(expect (let [rec (->MyRec "val")] (= rec (thaw (freeze rec)))))
 
-;;; Custom types
+;;; Extend to custom Type
 (defrecord MyType [data])
 (nippy/extend-freeze MyType 1 [x s] (.writeUTF s (:data x)))
 (expect Exception (thaw (freeze (->MyType "val"))))
 (expect (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s)))
             (let [type (->MyType "val")] (= type (thaw (freeze type))))))
 
-;;; Records (extend)
+;;; Extend to custom Record
+(defrecord MyRec [data])
 (expect (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "fast-" (:data x))))
             (nippy/extend-thaw 2 [s] (->MyRec (.readUTF s)))
             (= (->MyRec "fast-val") (thaw (freeze (->MyRec "val"))))))