luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev'

Browse source
This commit is contained in:
Peter Taoussanis 2014-02-16 18:57:31 +07:00
commit a9559ffc01
5 changed files with 114 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CHANGELOG.md
View file

@ -1,4 +1,4 @@
## v2.6.0-alpha2 / 2014-Jan-23
## v2.6.0-alpha4 / 2014-Feb-16
**WARNING**: This is an **EXPERIMENTAL early testing release** and **unsuitable for use in production**. Welcoming feedback on any issues, etc.!
@ -8,12 +8,17 @@
* New test suite added to ensure a 1-to-1 value->binary representation mapping for all core data types. This will be a guarantee kept going forward.
* New `:skip-header?` `freeze` option to freeze data without standard Nippy headers (can be useful in very performance sensitive environments).
* New benchmarks added, notably a Fressian comparison.
* Added experimental `freezable?` util fn to main ns.
* Added some property-based [simple-check](https://github.com/reiddraper/simple-check) roundtrip tests.
### Changes
* **BREAKING**: the experimental `Compressable-LZMA2` type has changed (less overhead).
* **DEPRECATED**: `freeze-to-stream!`, `thaw-from-stream!` are deprecated in favor of the more general `freeze-to-out!`, `thaw-from-in!`.
* **DEPRECATED**: `:legacy-mode` options. This was being used mainly for headerless freezing, so a new headerless mode is taking its place.
* Public utils now available for custom type extension: `write-bytes`, `write-biginteger`, `write-utf8`, `write-compact-long`, and respective readers.
* Now distinguish between `BigInteger` and `BigInt` on thawing (previously both thawed to `BigInt`s). (mlacorte).
### Fixes
* None.

8
project.clj
View file

@ -1,4 +1,4 @@
(defproject com.taoensso/nippy "2.6.0-alpha2"
(defproject com.taoensso/nippy "2.6.0-alpha3"
:description "Clojure serialization library"
:url "https://github.com/ptaoussanis/nippy"
:license {:name "Eclipse Public License"
@ -9,14 +9,14 @@
[org.tukaani/xz "1.4"]]
:profiles {:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]}
:1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]}
:1.6 {:dependencies [[org.clojure/clojure "1.6.0-alpha3"]]}
:1.6 {:dependencies [[org.clojure/clojure "1.6.0-beta1"]]}
:dev {:dependencies []}
:test {:jvm-opts ["-Xms1024m" ; Initial heap size
"-Xmx2048m" ; Max heap size
]
:dependencies [[expectations "1.4.56"]
[org.xerial.snappy/snappy-java "1.1.1-M1"]
[reiddraper/simple-check "0.5.3"]
[reiddraper/simple-check "0.5.6"]
[org.clojure/data.fressian "0.2.0"]]}
:bench {:dependencies [] :jvm-opts ^:replace ["-server"]}}
:aliases {"test-all" ["with-profile" "+test,+1.4:+test,+1.5:+test,+1.6" "expectations"]
@ -27,7 +27,7 @@
:plugins [[lein-expectations "0.0.8"]
[lein-autoexpect "1.2.1"]
[lein-ancient "0.5.4"]
[codox "0.6.6"]]
[codox "0.6.7"]]
:min-lein-version "2.0.0"
:global-vars {*warn-on-reflection* true}
:repositories

25
src/taoensso/nippy.clj
View file

@ -68,6 +68,7 @@
(def ^:const id-integer (int 42))
(def ^:const id-long (int 43))
(def ^:const id-bigint (int 44))
(def ^:const id-biginteger (int 45))
(def ^:const id-float (int 60))
(def ^:const id-double (int 61))
@ -246,8 +247,8 @@
;;
(freezer BigInt id-bigint (write-biginteger out (.toBigInteger x)))
(freezer BigInteger id-bigint (write-biginteger out x))
(freezer BigInt id-bigint (write-biginteger out (.toBigInteger x)))
(freezer BigInteger id-biginteger (write-biginteger out x))
(freezer Float id-float (.writeFloat out x))
(freezer Double id-double (.writeDouble out x))
@ -423,7 +424,8 @@
id-int-as-long (long (.readInt in))
;; id-compact-long (read-compact-long in)
id-bigint (bigint (read-biginteger in))
id-bigint (bigint (read-biginteger in))
id-biginteger (read-biginteger in)
id-float (.readFloat in)
id-double (.readDouble in)
@ -496,6 +498,10 @@
:as opts}]]
(let [headerless-meta (merge headerless-meta (:legacy-opts opts)) ; Deprecated
_ (assert (or (nil? headerless-meta)
(head-meta-id headerless-meta))
"Bad :headerless-meta (should be nil or a valid `head-meta` value)")
ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
try-thaw-data
(fn [data-ba {:keys [compressed? encrypted?] :as _head-or-headerless-meta}]
@ -593,8 +599,7 @@
(defrecord Compressable-LZMA2 [value])
(extend-freeze Compressable-LZMA2 128 [x out]
(let [[_ ^bytes ba] (-> (freeze (:value x) {:compressor nil})
(utils/ba-split 4))
(let [ba (freeze (:value x) {:skip-header? true :compressor nil})
ba-len (alength ba)
compress? (> ba-len 1024)]
(.writeBoolean out compress?)
@ -605,8 +610,10 @@
(extend-thaw 128 [in]
(let [compressed? (.readBoolean in)
ba (read-bytes in)]
(thaw (wrap-header ba {:compressed? compressed? :encrypted? false})
{:compressor compression/lzma2-compressor})))
(thaw ba {:compressor compression/lzma2-compressor
:headerless-meta {:version 1
:compressed? compressed?
:encrypted? false}})))
(comment
(->> (apply str (repeatedly 1000 rand))
@ -690,7 +697,9 @@
(dissoc stress-data :bytes :throwable :exception :ex-info :queue :queue-empty
:byte :stress-record))
;;;; Data recovery/analysis
;;;; Tools
(utils/defalias freezeable? utils/freezable?)
(defn inspect-ba "Alpha - subject to change."
[ba & [thaw-opts]]

81
src/taoensso/nippy/utils.clj
View file

@ -5,6 +5,16 @@
(:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable
ObjectOutputStream ObjectInputStream]))
(defmacro defalias
"Defines an alias for a var, preserving metadata. Adapted from
clojure.contrib/def.clj, Ref. http://goo.gl/xpjeH"
[name target & [doc]]
`(let [^clojure.lang.Var v# (var ~target)]
(alter-meta! (def ~name (.getRawRoot v#))
#(merge % (apply dissoc (meta v#) [:column :line :file :test :name])
(when-let [doc# ~doc] {:doc doc#})))
(var ~name)))
(defmacro case-eval
"Like `case` but evaluates test constants for their compile-time value."
[e & clauses]
@ -144,3 +154,74 @@
(time (dotimes [_ 10000] (serializable? (fn []))))
(time (dotimes [_ 10000] (readable? "Hello world")))
(time (dotimes [_ 10000] (readable? (fn [])))))
;;;;
(defn- is-coll?
"Checks for _explicit_ IPersistentCollection types with Nippy support."
[x]
(let [is? #(when (instance? % x) %)]
(or
(is? clojure.lang.APersistentVector)
(is? clojure.lang.APersistentMap)
(is? clojure.lang.APersistentSet)
(is? clojure.lang.PersistentList)
(is? clojure.lang.PersistentList$EmptyList) ; (type '())
(is? clojure.lang.PersistentQueue)
(is? clojure.lang.PersistentTreeSet)
(is? clojure.lang.PersistentTreeMap)
(is? clojure.lang.IRecord)
(is? clojure.lang.LazySeq)
;; (is? clojure.lang.ISeq)
)))
(defn freezable?
"Alpha - subject to change, may be buggy!
Returns truthy value iff Nippy supports de/serialization of given argument.
Conservative with default options.
`:allow-clojure-reader?` and `:allow-java-serializable?` options may be used
to also enable the relevant roundtrip fallback test(s). These tests are only
**moderately reliable** since they're cached by arg type and don't test for
pre/post serialization equality (there's no good general way of doing so)."
[x & [{:keys [allow-clojure-reader? allow-java-serializable?]}]]
(let [is? #(when (instance? % x) %)]
(if (is-coll? x)
(try
(when (every? freezable? x) (type x))
(catch Exception _ false))
(or
(is? clojure.lang.Keyword)
(is? java.lang.String)
(is? java.lang.Long)
(is? java.lang.Double)
(is? clojure.lang.BigInt)
(is? clojure.lang.Ratio)
(is? java.lang.Boolean)
(is? java.lang.Integer)
(is? java.lang.Short)
(is? java.lang.Byte)
(is? java.lang.Character)
(is? java.math.BigInteger)
(is? java.math.BigDecimal)
(is? #=(java.lang.Class/forName "[B"))
(is? java.util.Date)
(is? java.util.UUID)
(when (and allow-clojure-reader? (readable? x)) :clojure-reader)
(when (and allow-java-serializable?
;; Reports as true but is unreliable:
(not (is? clojure.lang.Fn))
(serializable? x)) :java-serializable)))))
(comment
(time (dotimes [_ 10000] (freezable? "hello")))
(freezable? [:a :b])
(freezable? [:a (fn [x] (* x x))])
(freezable? (.getBytes "foo"))
(freezable? (java.util.Date.) {:allow-clojure-reader? true})
(freezable? (Exception. "_") {:allow-clojure-reader? true})
(freezable? (Exception. "_") {:allow-java-serializable? true}))

7
test/taoensso/nippy/tests/main.clj
View file

@ -27,6 +27,11 @@
:password [:salted "p"]}))
test-data))
(expect ; Try roundtrip anything that simple-check can dream up
(:result (sc/quick-check 80 ; Time is n-non-linear
(sc-prop/for-all [val sc-gen/any]
(= val (nippy/thaw (nippy/freeze val)))))))
(expect AssertionError (thaw (freeze test-data {:password "malformed"})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})
@ -105,7 +110,7 @@
(let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)]
[result (vals bin->val)]))
;; (expect #(:result %) (qc-prop-bijection 120)) ; Time seems to be n-non-linear
;; (expect #(:result %) (qc-prop-bijection 120)) ; Time is n-non-linear
(expect #(:result %) (qc-prop-bijection 80))
;;;; Benchmarks