diff --git a/CHANGELOG.md b/CHANGELOG.md index 7af2119..23a4bca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v2.6.0-alpha2 / 2014-Jan-23 +## v2.6.0-alpha4 / 2014-Feb-16 **WARNING**: This is an **EXPERIMENTAL early testing release** and **unsuitable for use in production**. Welcoming feedback on any issues, etc.! @@ -8,12 +8,17 @@ * New test suite added to ensure a 1-to-1 value->binary representation mapping for all core data types. This will be a guarantee kept going forward. * New `:skip-header?` `freeze` option to freeze data without standard Nippy headers (can be useful in very performance sensitive environments). * New benchmarks added, notably a Fressian comparison. + * Added experimental `freezable?` util fn to main ns. + * Added some property-based [simple-check](https://github.com/reiddraper/simple-check) roundtrip tests. + ### Changes * **BREAKING**: the experimental `Compressable-LZMA2` type has changed (less overhead). * **DEPRECATED**: `freeze-to-stream!`, `thaw-from-stream!` are deprecated in favor of the more general `freeze-to-out!`, `thaw-from-in!`. * **DEPRECATED**: `:legacy-mode` options. This was being used mainly for headerless freezing, so a new headerless mode is taking its place. * Public utils now available for custom type extension: `write-bytes`, `write-biginteger`, `write-utf8`, `write-compact-long`, and respective readers. + * Now distinguish between `BigInteger` and `BigInt` on thawing (previously both thawed to `BigInt`s). (mlacorte). + ### Fixes * None. diff --git a/project.clj b/project.clj index 6554c29..20e5d45 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.6.0-alpha2" +(defproject com.taoensso/nippy "2.6.0-alpha3" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License" @@ -9,14 +9,14 @@ [org.tukaani/xz "1.4"]] :profiles {:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} - :1.6 {:dependencies [[org.clojure/clojure "1.6.0-alpha3"]]} + :1.6 {:dependencies [[org.clojure/clojure "1.6.0-beta1"]]} :dev {:dependencies []} :test {:jvm-opts ["-Xms1024m" ; Initial heap size "-Xmx2048m" ; Max heap size ] :dependencies [[expectations "1.4.56"] [org.xerial.snappy/snappy-java "1.1.1-M1"] - [reiddraper/simple-check "0.5.3"] + [reiddraper/simple-check "0.5.6"] [org.clojure/data.fressian "0.2.0"]]} :bench {:dependencies [] :jvm-opts ^:replace ["-server"]}} :aliases {"test-all" ["with-profile" "+test,+1.4:+test,+1.5:+test,+1.6" "expectations"] @@ -27,7 +27,7 @@ :plugins [[lein-expectations "0.0.8"] [lein-autoexpect "1.2.1"] [lein-ancient "0.5.4"] - [codox "0.6.6"]] + [codox "0.6.7"]] :min-lein-version "2.0.0" :global-vars {*warn-on-reflection* true} :repositories diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 00b47c4..0bcdceb 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -68,6 +68,7 @@ (def ^:const id-integer (int 42)) (def ^:const id-long (int 43)) (def ^:const id-bigint (int 44)) + (def ^:const id-biginteger (int 45)) (def ^:const id-float (int 60)) (def ^:const id-double (int 61)) @@ -246,8 +247,8 @@ ;; -(freezer BigInt id-bigint (write-biginteger out (.toBigInteger x))) -(freezer BigInteger id-bigint (write-biginteger out x)) +(freezer BigInt id-bigint (write-biginteger out (.toBigInteger x))) +(freezer BigInteger id-biginteger (write-biginteger out x)) (freezer Float id-float (.writeFloat out x)) (freezer Double id-double (.writeDouble out x)) @@ -423,7 +424,8 @@ id-int-as-long (long (.readInt in)) ;; id-compact-long (read-compact-long in) - id-bigint (bigint (read-biginteger in)) + id-bigint (bigint (read-biginteger in)) + id-biginteger (read-biginteger in) id-float (.readFloat in) id-double (.readDouble in) @@ -496,6 +498,10 @@ :as opts}]] (let [headerless-meta (merge headerless-meta (:legacy-opts opts)) ; Deprecated + _ (assert (or (nil? headerless-meta) + (head-meta-id headerless-meta)) + "Bad :headerless-meta (should be nil or a valid `head-meta` value)") + ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e))) try-thaw-data (fn [data-ba {:keys [compressed? encrypted?] :as _head-or-headerless-meta}] @@ -593,8 +599,7 @@ (defrecord Compressable-LZMA2 [value]) (extend-freeze Compressable-LZMA2 128 [x out] - (let [[_ ^bytes ba] (-> (freeze (:value x) {:compressor nil}) - (utils/ba-split 4)) + (let [ba (freeze (:value x) {:skip-header? true :compressor nil}) ba-len (alength ba) compress? (> ba-len 1024)] (.writeBoolean out compress?) @@ -605,8 +610,10 @@ (extend-thaw 128 [in] (let [compressed? (.readBoolean in) ba (read-bytes in)] - (thaw (wrap-header ba {:compressed? compressed? :encrypted? false}) - {:compressor compression/lzma2-compressor}))) + (thaw ba {:compressor compression/lzma2-compressor + :headerless-meta {:version 1 + :compressed? compressed? + :encrypted? false}}))) (comment (->> (apply str (repeatedly 1000 rand)) @@ -690,7 +697,9 @@ (dissoc stress-data :bytes :throwable :exception :ex-info :queue :queue-empty :byte :stress-record)) -;;;; Data recovery/analysis +;;;; Tools + +(utils/defalias freezeable? utils/freezable?) (defn inspect-ba "Alpha - subject to change." [ba & [thaw-opts]] diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 26ba470..84b152e 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -5,6 +5,16 @@ (:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable ObjectOutputStream ObjectInputStream])) +(defmacro defalias + "Defines an alias for a var, preserving metadata. Adapted from + clojure.contrib/def.clj, Ref. http://goo.gl/xpjeH" + [name target & [doc]] + `(let [^clojure.lang.Var v# (var ~target)] + (alter-meta! (def ~name (.getRawRoot v#)) + #(merge % (apply dissoc (meta v#) [:column :line :file :test :name]) + (when-let [doc# ~doc] {:doc doc#}))) + (var ~name))) + (defmacro case-eval "Like `case` but evaluates test constants for their compile-time value." [e & clauses] @@ -144,3 +154,74 @@ (time (dotimes [_ 10000] (serializable? (fn [])))) (time (dotimes [_ 10000] (readable? "Hello world"))) (time (dotimes [_ 10000] (readable? (fn []))))) + +;;;; + +(defn- is-coll? + "Checks for _explicit_ IPersistentCollection types with Nippy support." + [x] + (let [is? #(when (instance? % x) %)] + (or + (is? clojure.lang.APersistentVector) + (is? clojure.lang.APersistentMap) + (is? clojure.lang.APersistentSet) + (is? clojure.lang.PersistentList) + (is? clojure.lang.PersistentList$EmptyList) ; (type '()) + (is? clojure.lang.PersistentQueue) + (is? clojure.lang.PersistentTreeSet) + (is? clojure.lang.PersistentTreeMap) + (is? clojure.lang.IRecord) + (is? clojure.lang.LazySeq) + ;; (is? clojure.lang.ISeq) + ))) + +(defn freezable? + "Alpha - subject to change, may be buggy! + Returns truthy value iff Nippy supports de/serialization of given argument. + Conservative with default options. + + `:allow-clojure-reader?` and `:allow-java-serializable?` options may be used + to also enable the relevant roundtrip fallback test(s). These tests are only + **moderately reliable** since they're cached by arg type and don't test for + pre/post serialization equality (there's no good general way of doing so)." + [x & [{:keys [allow-clojure-reader? allow-java-serializable?]}]] + (let [is? #(when (instance? % x) %)] + (if (is-coll? x) + (try + (when (every? freezable? x) (type x)) + (catch Exception _ false)) + (or + (is? clojure.lang.Keyword) + (is? java.lang.String) + (is? java.lang.Long) + (is? java.lang.Double) + + (is? clojure.lang.BigInt) + (is? clojure.lang.Ratio) + + (is? java.lang.Boolean) + (is? java.lang.Integer) + (is? java.lang.Short) + (is? java.lang.Byte) + (is? java.lang.Character) + (is? java.math.BigInteger) + (is? java.math.BigDecimal) + (is? #=(java.lang.Class/forName "[B")) + + (is? java.util.Date) + (is? java.util.UUID) + + (when (and allow-clojure-reader? (readable? x)) :clojure-reader) + (when (and allow-java-serializable? + ;; Reports as true but is unreliable: + (not (is? clojure.lang.Fn)) + (serializable? x)) :java-serializable))))) + +(comment + (time (dotimes [_ 10000] (freezable? "hello"))) + (freezable? [:a :b]) + (freezable? [:a (fn [x] (* x x))]) + (freezable? (.getBytes "foo")) + (freezable? (java.util.Date.) {:allow-clojure-reader? true}) + (freezable? (Exception. "_") {:allow-clojure-reader? true}) + (freezable? (Exception. "_") {:allow-java-serializable? true})) diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index a033981..01f823a 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -27,6 +27,11 @@ :password [:salted "p"]})) test-data)) +(expect ; Try roundtrip anything that simple-check can dream up + (:result (sc/quick-check 80 ; Time is n-non-linear + (sc-prop/for-all [val sc-gen/any] + (= val (nippy/thaw (nippy/freeze val))))))) + (expect AssertionError (thaw (freeze test-data {:password "malformed"}))) (expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) (expect Exception (thaw (freeze test-data {:password [:salted "p"]}) @@ -105,7 +110,7 @@ (let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)] [result (vals bin->val)])) -;; (expect #(:result %) (qc-prop-bijection 120)) ; Time seems to be n-non-linear +;; (expect #(:result %) (qc-prop-bijection 120)) ; Time is n-non-linear (expect #(:result %) (qc-prop-bijection 80)) ;;;; Benchmarks