From 17c81618168ed9f8063b56bd502fbbd0111528ed Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 30 Jan 2014 15:59:17 +0700 Subject: [PATCH 1/8] Deps: simple-check 0.5.6, --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 6554c29..f801042 100644 --- a/project.clj +++ b/project.clj @@ -16,7 +16,7 @@ ] :dependencies [[expectations "1.4.56"] [org.xerial.snappy/snappy-java "1.1.1-M1"] - [reiddraper/simple-check "0.5.3"] + [reiddraper/simple-check "0.5.6"] [org.clojure/data.fressian "0.2.0"]]} :bench {:dependencies [] :jvm-opts ^:replace ["-server"]}} :aliases {"test-all" ["with-profile" "+test,+1.4:+test,+1.5:+test,+1.6" "expectations"] From e2847f348f3751d24ab66c89c0502e843ec6b6cb Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 30 Jan 2014 16:14:37 +0700 Subject: [PATCH 2/8] [#38]: Distinguish between BigInt/BigInteger on thawing (mlacorte) --- CHANGELOG.md | 4 ++++ src/taoensso/nippy.clj | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7af2119..c51a97e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Pending + * Now distinguish between `BigInteger` and `BigInt` on thawing (previously both thawed to `BigInt`s). (mlacorte). + + ## v2.6.0-alpha2 / 2014-Jan-23 **WARNING**: This is an **EXPERIMENTAL early testing release** and **unsuitable for use in production**. Welcoming feedback on any issues, etc.! diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 00b47c4..3d72a0c 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -68,6 +68,7 @@ (def ^:const id-integer (int 42)) (def ^:const id-long (int 43)) (def ^:const id-bigint (int 44)) + (def ^:const id-biginteger (int 45)) (def ^:const id-float (int 60)) (def ^:const id-double (int 61)) @@ -246,8 +247,8 @@ ;; -(freezer BigInt id-bigint (write-biginteger out (.toBigInteger x))) -(freezer BigInteger id-bigint (write-biginteger out x)) +(freezer BigInt id-bigint (write-biginteger out (.toBigInteger x))) +(freezer BigInteger id-biginteger (write-biginteger out x)) (freezer Float id-float (.writeFloat out x)) (freezer Double id-double (.writeDouble out x)) @@ -423,7 +424,8 @@ id-int-as-long (long (.readInt in)) ;; id-compact-long (read-compact-long in) - id-bigint (bigint (read-biginteger in)) + id-bigint (bigint (read-biginteger in)) + id-biginteger (read-biginteger in) id-float (.readFloat in) id-double (.readDouble in) From 0b56746a61bf2384d97863cf929a29dab87e3d36 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 9 Feb 2014 18:29:16 +0700 Subject: [PATCH 3/8] Mod (experimental) `Compressable-LZMA2` to use new :skip-header? mode, add :headerless-meta assertion --- src/taoensso/nippy.clj | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 3d72a0c..0c69606 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -498,6 +498,10 @@ :as opts}]] (let [headerless-meta (merge headerless-meta (:legacy-opts opts)) ; Deprecated + _ (assert (or (nil? headerless-meta) + (head-meta-id headerless-meta)) + "Bad :headerless-meta (should be nil or a valid `head-meta` value)") + ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e))) try-thaw-data (fn [data-ba {:keys [compressed? encrypted?] :as _head-or-headerless-meta}] @@ -595,8 +599,7 @@ (defrecord Compressable-LZMA2 [value]) (extend-freeze Compressable-LZMA2 128 [x out] - (let [[_ ^bytes ba] (-> (freeze (:value x) {:compressor nil}) - (utils/ba-split 4)) + (let [ba (freeze (:value x) {:skip-header? true :compressor nil}) ba-len (alength ba) compress? (> ba-len 1024)] (.writeBoolean out compress?) @@ -607,8 +610,10 @@ (extend-thaw 128 [in] (let [compressed? (.readBoolean in) ba (read-bytes in)] - (thaw (wrap-header ba {:compressed? compressed? :encrypted? false}) - {:compressor compression/lzma2-compressor}))) + (thaw ba {:compressor compression/lzma2-compressor + :headerless-meta {:version 1 + :compressed? compressed? + :encrypted? false}}))) (comment (->> (apply str (repeatedly 1000 rand)) From 58d882b12b9b36216cb14870599f5f9fea8f1567 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 9 Feb 2014 18:30:06 +0700 Subject: [PATCH 4/8] v2.6.0-alpha3 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index f801042..5f6cb6e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.6.0-alpha2" +(defproject com.taoensso/nippy "2.6.0-alpha3" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License" From 42b366a432e3d306d82ef50c5b3c042ec0b4af69 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Fri, 14 Feb 2014 23:06:53 +0700 Subject: [PATCH 5/8] Experimental `freezable?` fn --- src/taoensso/nippy.clj | 4 +- src/taoensso/nippy/utils.clj | 81 ++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 0c69606..0bcdceb 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -697,7 +697,9 @@ (dissoc stress-data :bytes :throwable :exception :ex-info :queue :queue-empty :byte :stress-record)) -;;;; Data recovery/analysis +;;;; Tools + +(utils/defalias freezeable? utils/freezable?) (defn inspect-ba "Alpha - subject to change." [ba & [thaw-opts]] diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 26ba470..84b152e 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -5,6 +5,16 @@ (:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable ObjectOutputStream ObjectInputStream])) +(defmacro defalias + "Defines an alias for a var, preserving metadata. Adapted from + clojure.contrib/def.clj, Ref. http://goo.gl/xpjeH" + [name target & [doc]] + `(let [^clojure.lang.Var v# (var ~target)] + (alter-meta! (def ~name (.getRawRoot v#)) + #(merge % (apply dissoc (meta v#) [:column :line :file :test :name]) + (when-let [doc# ~doc] {:doc doc#}))) + (var ~name))) + (defmacro case-eval "Like `case` but evaluates test constants for their compile-time value." [e & clauses] @@ -144,3 +154,74 @@ (time (dotimes [_ 10000] (serializable? (fn [])))) (time (dotimes [_ 10000] (readable? "Hello world"))) (time (dotimes [_ 10000] (readable? (fn []))))) + +;;;; + +(defn- is-coll? + "Checks for _explicit_ IPersistentCollection types with Nippy support." + [x] + (let [is? #(when (instance? % x) %)] + (or + (is? clojure.lang.APersistentVector) + (is? clojure.lang.APersistentMap) + (is? clojure.lang.APersistentSet) + (is? clojure.lang.PersistentList) + (is? clojure.lang.PersistentList$EmptyList) ; (type '()) + (is? clojure.lang.PersistentQueue) + (is? clojure.lang.PersistentTreeSet) + (is? clojure.lang.PersistentTreeMap) + (is? clojure.lang.IRecord) + (is? clojure.lang.LazySeq) + ;; (is? clojure.lang.ISeq) + ))) + +(defn freezable? + "Alpha - subject to change, may be buggy! + Returns truthy value iff Nippy supports de/serialization of given argument. + Conservative with default options. + + `:allow-clojure-reader?` and `:allow-java-serializable?` options may be used + to also enable the relevant roundtrip fallback test(s). These tests are only + **moderately reliable** since they're cached by arg type and don't test for + pre/post serialization equality (there's no good general way of doing so)." + [x & [{:keys [allow-clojure-reader? allow-java-serializable?]}]] + (let [is? #(when (instance? % x) %)] + (if (is-coll? x) + (try + (when (every? freezable? x) (type x)) + (catch Exception _ false)) + (or + (is? clojure.lang.Keyword) + (is? java.lang.String) + (is? java.lang.Long) + (is? java.lang.Double) + + (is? clojure.lang.BigInt) + (is? clojure.lang.Ratio) + + (is? java.lang.Boolean) + (is? java.lang.Integer) + (is? java.lang.Short) + (is? java.lang.Byte) + (is? java.lang.Character) + (is? java.math.BigInteger) + (is? java.math.BigDecimal) + (is? #=(java.lang.Class/forName "[B")) + + (is? java.util.Date) + (is? java.util.UUID) + + (when (and allow-clojure-reader? (readable? x)) :clojure-reader) + (when (and allow-java-serializable? + ;; Reports as true but is unreliable: + (not (is? clojure.lang.Fn)) + (serializable? x)) :java-serializable))))) + +(comment + (time (dotimes [_ 10000] (freezable? "hello"))) + (freezable? [:a :b]) + (freezable? [:a (fn [x] (* x x))]) + (freezable? (.getBytes "foo")) + (freezable? (java.util.Date.) {:allow-clojure-reader? true}) + (freezable? (Exception. "_") {:allow-clojure-reader? true}) + (freezable? (Exception. "_") {:allow-java-serializable? true})) From 557cf2eda79c9784f75fdd8ea53573deb13d70e5 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 16 Feb 2014 18:49:33 +0700 Subject: [PATCH 6/8] Add simple-check based roundtrip test --- test/taoensso/nippy/tests/main.clj | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index a033981..01f823a 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -27,6 +27,11 @@ :password [:salted "p"]})) test-data)) +(expect ; Try roundtrip anything that simple-check can dream up + (:result (sc/quick-check 80 ; Time is n-non-linear + (sc-prop/for-all [val sc-gen/any] + (= val (nippy/thaw (nippy/freeze val))))))) + (expect AssertionError (thaw (freeze test-data {:password "malformed"}))) (expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) (expect Exception (thaw (freeze test-data {:password [:salted "p"]}) @@ -105,7 +110,7 @@ (let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)] [result (vals bin->val)])) -;; (expect #(:result %) (qc-prop-bijection 120)) ; Time seems to be n-non-linear +;; (expect #(:result %) (qc-prop-bijection 120)) ; Time is n-non-linear (expect #(:result %) (qc-prop-bijection 80)) ;;;; Benchmarks From 3b8e426c435e91e0a7a5b4a325070d4f09a2fdb0 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 16 Feb 2014 18:51:00 +0700 Subject: [PATCH 7/8] Deps: Clojure 1.6.0-beta1 (for tests), codox 0.6.7 --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 5f6cb6e..20e5d45 100644 --- a/project.clj +++ b/project.clj @@ -9,7 +9,7 @@ [org.tukaani/xz "1.4"]] :profiles {:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} - :1.6 {:dependencies [[org.clojure/clojure "1.6.0-alpha3"]]} + :1.6 {:dependencies [[org.clojure/clojure "1.6.0-beta1"]]} :dev {:dependencies []} :test {:jvm-opts ["-Xms1024m" ; Initial heap size "-Xmx2048m" ; Max heap size @@ -27,7 +27,7 @@ :plugins [[lein-expectations "0.0.8"] [lein-autoexpect "1.2.1"] [lein-ancient "0.5.4"] - [codox "0.6.6"]] + [codox "0.6.7"]] :min-lein-version "2.0.0" :global-vars {*warn-on-reflection* true} :repositories From 6a883cf1d83bb3b8b437fa9be2f1923c149660a0 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 16 Feb 2014 18:54:08 +0700 Subject: [PATCH 8/8] v2.6.0-alpha4 --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c51a97e..23a4bca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,4 @@ -## Pending - * Now distinguish between `BigInteger` and `BigInt` on thawing (previously both thawed to `BigInt`s). (mlacorte). - - -## v2.6.0-alpha2 / 2014-Jan-23 +## v2.6.0-alpha4 / 2014-Feb-16 **WARNING**: This is an **EXPERIMENTAL early testing release** and **unsuitable for use in production**. Welcoming feedback on any issues, etc.! @@ -12,12 +8,17 @@ * New test suite added to ensure a 1-to-1 value->binary representation mapping for all core data types. This will be a guarantee kept going forward. * New `:skip-header?` `freeze` option to freeze data without standard Nippy headers (can be useful in very performance sensitive environments). * New benchmarks added, notably a Fressian comparison. + * Added experimental `freezable?` util fn to main ns. + * Added some property-based [simple-check](https://github.com/reiddraper/simple-check) roundtrip tests. + ### Changes * **BREAKING**: the experimental `Compressable-LZMA2` type has changed (less overhead). * **DEPRECATED**: `freeze-to-stream!`, `thaw-from-stream!` are deprecated in favor of the more general `freeze-to-out!`, `thaw-from-in!`. * **DEPRECATED**: `:legacy-mode` options. This was being used mainly for headerless freezing, so a new headerless mode is taking its place. * Public utils now available for custom type extension: `write-bytes`, `write-biginteger`, `write-utf8`, `write-compact-long`, and respective readers. + * Now distinguish between `BigInteger` and `BigInt` on thawing (previously both thawed to `BigInt`s). (mlacorte). + ### Fixes * None.