Merge branch 'dev'

CHANGELOG.md

@@ -1,4 +1,4 @@
-## v2.6.0-alpha2 / 2014-Jan-23
+## v2.6.0-alpha4 / 2014-Feb-16
 
 **WARNING**: This is an **EXPERIMENTAL early testing release** and **unsuitable for use in production**. Welcoming feedback on any issues, etc.!
 
@@ -8,12 +8,17 @@
  * New test suite added to ensure a 1-to-1 value->binary representation mapping for all core data types. This will be a guarantee kept going forward.
  * New `:skip-header?` `freeze` option to freeze data without standard Nippy headers (can be useful in very performance sensitive environments).
  * New benchmarks added, notably a Fressian comparison.
+ * Added experimental `freezable?` util fn to main ns.
+ * Added some property-based [simple-check](https://github.com/reiddraper/simple-check) roundtrip tests.
+
 
 ### Changes
  * **BREAKING**: the experimental `Compressable-LZMA2` type has changed (less overhead).
  * **DEPRECATED**: `freeze-to-stream!`, `thaw-from-stream!` are deprecated in favor of the more general `freeze-to-out!`, `thaw-from-in!`.
  * **DEPRECATED**: `:legacy-mode` options. This was being used mainly for headerless freezing, so a new headerless mode is taking its place.
  * Public utils now available for custom type extension: `write-bytes`, `write-biginteger`, `write-utf8`, `write-compact-long`, and respective readers.
+ * Now distinguish between `BigInteger` and `BigInt` on thawing (previously both thawed to `BigInt`s). (mlacorte).
+
 
 ### Fixes
  * None.

project.clj

@@ -1,4 +1,4 @@
-(defproject com.taoensso/nippy "2.6.0-alpha2"
+(defproject com.taoensso/nippy "2.6.0-alpha3"
   :description "Clojure serialization library"
   :url "https://github.com/ptaoussanis/nippy"
   :license {:name "Eclipse Public License"
@@ -9,14 +9,14 @@
                  [org.tukaani/xz           "1.4"]]
   :profiles {:1.4   {:dependencies [[org.clojure/clojure "1.4.0"]]}
              :1.5   {:dependencies [[org.clojure/clojure "1.5.1"]]}
-             :1.6   {:dependencies [[org.clojure/clojure "1.6.0-alpha3"]]}
+             :1.6   {:dependencies [[org.clojure/clojure "1.6.0-beta1"]]}
              :dev   {:dependencies []}
              :test  {:jvm-opts ["-Xms1024m" ; Initial heap size
                                 "-Xmx2048m" ; Max heap size
                                 ]
                      :dependencies [[expectations                  "1.4.56"]
                                     [org.xerial.snappy/snappy-java "1.1.1-M1"]
-                                    [reiddraper/simple-check       "0.5.3"]
+                                    [reiddraper/simple-check       "0.5.6"]
                                     [org.clojure/data.fressian     "0.2.0"]]}
              :bench {:dependencies [] :jvm-opts ^:replace ["-server"]}}
   :aliases {"test-all"    ["with-profile" "+test,+1.4:+test,+1.5:+test,+1.6" "expectations"]
@@ -27,7 +27,7 @@
   :plugins [[lein-expectations "0.0.8"]
             [lein-autoexpect   "1.2.1"]
             [lein-ancient      "0.5.4"]
-            [codox             "0.6.6"]]
+            [codox             "0.6.7"]]
   :min-lein-version "2.0.0"
   :global-vars {*warn-on-reflection* true}
   :repositories

src/taoensso/nippy.clj

@@ -68,6 +68,7 @@
   (def ^:const id-integer    (int 42))
   (def ^:const id-long       (int 43))
   (def ^:const id-bigint     (int 44))
+  (def ^:const id-biginteger (int 45))
 
   (def ^:const id-float      (int 60))
   (def ^:const id-double     (int 61))
@@ -247,7 +248,7 @@
 ;;
 
 (freezer BigInt     id-bigint     (write-biginteger out (.toBigInteger x)))
-(freezer BigInteger id-bigint  (write-biginteger out x))
+(freezer BigInteger id-biginteger (write-biginteger out x))
 
 (freezer Float      id-float   (.writeFloat  out x))
 (freezer Double     id-double  (.writeDouble out x))
@@ -424,6 +425,7 @@
         ;; id-compact-long  (read-compact-long in)
 
         id-bigint     (bigint (read-biginteger in))
+        id-biginteger (read-biginteger in)
 
         id-float  (.readFloat  in)
         id-double (.readDouble in)
@@ -496,6 +498,10 @@
                  :as   opts}]]
 
   (let [headerless-meta (merge headerless-meta (:legacy-opts opts)) ; Deprecated
+        _ (assert (or (nil? headerless-meta)
+                      (head-meta-id headerless-meta))
+                  "Bad :headerless-meta (should be nil or a valid `head-meta` value)")
+
         ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
         try-thaw-data
         (fn [data-ba {:keys [compressed? encrypted?] :as _head-or-headerless-meta}]
@@ -593,8 +599,7 @@
 
 (defrecord Compressable-LZMA2 [value])
 (extend-freeze Compressable-LZMA2 128 [x out]
-  (let [[_ ^bytes ba] (-> (freeze (:value x) {:compressor nil})
+  (let [ba (freeze (:value x) {:skip-header? true :compressor nil})
-                          (utils/ba-split 4))
         ba-len    (alength ba)
         compress? (> ba-len 1024)]
     (.writeBoolean out compress?)
@@ -605,8 +610,10 @@
 (extend-thaw 128 [in]
   (let [compressed? (.readBoolean in)
         ba          (read-bytes in)]
-    (thaw (wrap-header ba {:compressed? compressed? :encrypted? false})
+    (thaw ba {:compressor compression/lzma2-compressor
-          {:compressor compression/lzma2-compressor})))
+              :headerless-meta {:version     1
+                                :compressed? compressed?
+                                :encrypted?  false}})))
 
 (comment
   (->> (apply str (repeatedly 1000 rand))
@@ -690,7 +697,9 @@
   (dissoc stress-data :bytes :throwable :exception :ex-info :queue :queue-empty
                       :byte :stress-record))
 
-;;;; Data recovery/analysis
+;;;; Tools
+
+(utils/defalias freezeable? utils/freezable?)
 
 (defn inspect-ba "Alpha - subject to change."
   [ba & [thaw-opts]]

src/taoensso/nippy/utils.clj

@@ -5,6 +5,16 @@
   (:import  [java.io ByteArrayInputStream ByteArrayOutputStream Serializable
              ObjectOutputStream ObjectInputStream]))
 
+(defmacro defalias
+  "Defines an alias for a var, preserving metadata. Adapted from
+  clojure.contrib/def.clj, Ref. http://goo.gl/xpjeH"
+  [name target & [doc]]
+  `(let [^clojure.lang.Var v# (var ~target)]
+     (alter-meta! (def ~name (.getRawRoot v#))
+                  #(merge % (apply dissoc (meta v#) [:column :line :file :test :name])
+                            (when-let [doc# ~doc] {:doc doc#})))
+     (var ~name)))
+
 (defmacro case-eval
   "Like `case` but evaluates test constants for their compile-time value."
   [e & clauses]
@@ -144,3 +154,74 @@
   (time (dotimes [_ 10000] (serializable? (fn []))))
   (time (dotimes [_ 10000] (readable? "Hello world")))
   (time (dotimes [_ 10000] (readable? (fn [])))))
+
+;;;;
+
+(defn- is-coll?
+  "Checks for _explicit_ IPersistentCollection types with Nippy support."
+  [x]
+  (let [is? #(when (instance? % x) %)]
+    (or
+     (is? clojure.lang.APersistentVector)
+     (is? clojure.lang.APersistentMap)
+     (is? clojure.lang.APersistentSet)
+     (is? clojure.lang.PersistentList)
+     (is? clojure.lang.PersistentList$EmptyList) ; (type '())
+     (is? clojure.lang.PersistentQueue)
+     (is? clojure.lang.PersistentTreeSet)
+     (is? clojure.lang.PersistentTreeMap)
+     (is? clojure.lang.IRecord)
+     (is? clojure.lang.LazySeq)
+     ;; (is? clojure.lang.ISeq)
+     )))
+
+(defn freezable?
+  "Alpha - subject to change, may be buggy!
+  Returns truthy value iff Nippy supports de/serialization of given argument.
+  Conservative with default options.
+
+  `:allow-clojure-reader?` and `:allow-java-serializable?` options may be used
+  to also enable the relevant roundtrip fallback test(s). These tests are only
+  **moderately reliable** since they're cached by arg type and don't test for
+  pre/post serialization equality (there's no good general way of doing so)."
+  [x & [{:keys [allow-clojure-reader? allow-java-serializable?]}]]
+  (let [is? #(when (instance? % x) %)]
+    (if (is-coll? x)
+      (try
+        (when (every? freezable? x) (type x))
+        (catch Exception _ false))
+      (or
+       (is? clojure.lang.Keyword)
+       (is? java.lang.String)
+       (is? java.lang.Long)
+       (is? java.lang.Double)
+
+       (is? clojure.lang.BigInt)
+       (is? clojure.lang.Ratio)
+
+       (is? java.lang.Boolean)
+       (is? java.lang.Integer)
+       (is? java.lang.Short)
+       (is? java.lang.Byte)
+       (is? java.lang.Character)
+       (is? java.math.BigInteger)
+       (is? java.math.BigDecimal)
+       (is? #=(java.lang.Class/forName "[B"))
+
+       (is? java.util.Date)
+       (is? java.util.UUID)
+
+       (when (and allow-clojure-reader? (readable? x)) :clojure-reader)
+       (when (and allow-java-serializable?
+                  ;; Reports as true but is unreliable:
+                  (not (is? clojure.lang.Fn))
+                  (serializable? x)) :java-serializable)))))
+
+(comment
+  (time (dotimes [_ 10000] (freezable? "hello")))
+  (freezable? [:a :b])
+  (freezable? [:a (fn [x] (* x x))])
+  (freezable? (.getBytes "foo"))
+  (freezable? (java.util.Date.) {:allow-clojure-reader?    true})
+  (freezable? (Exception. "_")  {:allow-clojure-reader?    true})
+  (freezable? (Exception. "_")  {:allow-java-serializable? true}))

test/taoensso/nippy/tests/main.clj

@@ -27,6 +27,11 @@
                                      :password [:salted "p"]}))
                    test-data))
 
+(expect ; Try roundtrip anything that simple-check can dream up
+ (:result (sc/quick-check 80 ; Time is n-non-linear
+            (sc-prop/for-all [val sc-gen/any]
+              (= val (nippy/thaw (nippy/freeze val)))))))
+
 (expect AssertionError (thaw (freeze test-data {:password "malformed"})))
 (expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
 (expect Exception (thaw (freeze test-data {:password [:salted "p"]})
@@ -105,7 +110,7 @@
   (let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)]
     [result (vals bin->val)]))
 
-;; (expect #(:result %) (qc-prop-bijection 120))  ; Time seems to be n-non-linear
+;; (expect #(:result %) (qc-prop-bijection 120)) ; Time is n-non-linear
 (expect #(:result %) (qc-prop-bijection 80))
 
 ;;;; Benchmarks