luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NB: Refactor thaw v1 compatibility support

Browse source
This commit is contained in:
Peter Taoussanis 2015-10-06 13:32:43 +07:00
parent 9c8adfe513
commit 0905b96ca6
1 changed files with 76 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
src/taoensso/nippy.clj
View file

@ -407,7 +407,8 @@
:or {compressor :auto :or {compressor :auto
encryptor aes128-encryptor} encryptor aes128-encryptor}
:as opts}] :as opts}]
(let [skip-header? (:skip-header? opts) ; Want this *un*documented (let [;; Intentionally undocumented:
no-header? (or (:no-header? opts) (:skip-header? opts))
encryptor (when password encryptor) encryptor (when password encryptor)
zero-copy-mode? (and (nil? compressor) (nil? encryptor)) zero-copy-mode? (and (nil? compressor) (nil? encryptor))
baos (ByteArrayOutputStream. 64) baos (ByteArrayOutputStream. 64)
@ -415,7 +416,7 @@
(if zero-copy-mode? (if zero-copy-mode?
(do ; Optimized case (do ; Optimized case
(when-not skip-header? ; Avoid `wrap-header`'s array copy: (when-not no-header? ; Avoid `wrap-header`'s array copy:
(let [head-ba (get-head-ba {:compressor-id nil :encryptor-id nil})] (let [head-ba (get-head-ba {:compressor-id nil :encryptor-id nil})]
(.write dos head-ba 0 4))) (.write dos head-ba 0 4)))
(freeze-to-out! dos x) (freeze-to-out! dos x)
@ -427,7 +428,7 @@
compressor compressor
(if (identical? compressor :auto) (if (identical? compressor :auto)
(if skip-header? (if no-header?
lz4-compressor lz4-compressor
(*default-freeze-compressor-selector* ba)) (*default-freeze-compressor-selector* ba))
(if (fn? compressor) (if (fn? compressor)
@ -438,7 +439,7 @@
ba (if compressor (compress compressor ba) ba) ba (if compressor (compress compressor ba) ba)
ba (if encryptor (encrypt encryptor password ba) ba)] ba (if encryptor (encrypt encryptor password ba) ba)]
(if skip-header? (if no-header?
ba ba
(wrap-header ba (wrap-header ba
{:compressor-id (when-let [c compressor] {:compressor-id (when-let [c compressor]
@ -654,6 +655,12 @@
(throw (ex-info (format "Unrecognized :auto encryptor id: %s" encryptor-id) (throw (ex-info (format "Unrecognized :auto encryptor id: %s" encryptor-id)
{:encryptor-id encryptor-id})))) {:encryptor-id encryptor-id}))))
(def ^:private err-msg-unknown-thaw-failure
"Decryption/decompression failure, or data unfrozen/damaged.")
(def ^:private err-msg-unrecognized-header
"Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?")
(defn thaw (defn thaw
"Deserializes a frozen object from given byte array to its original Clojure "Deserializes a frozen object from given byte array to its original Clojure
data type. To thaw custom types, extend the Clojure reader or see `extend-thaw`. data type. To thaw custom types, extend the Clojure reader or see `extend-thaw`.
@ -669,7 +676,7 @@
([ba] (thaw ba nil)) ([ba] (thaw ba nil))
([^bytes ba ([^bytes ba
{:keys [v1-compatibility? compressor encryptor password] {:keys [v1-compatibility? compressor encryptor password no-header?]
:or {compressor :auto :or {compressor :auto
encryptor :auto} encryptor :auto}
:as opts}] :as opts}]
@ -677,13 +684,17 @@
(assert (not (:headerless-meta opts)) (assert (not (:headerless-meta opts))
":headerless-meta `thaw` opt removed in Nippy v2.7+") ":headerless-meta `thaw` opt removed in Nippy v2.7+")
(let [ex (fn [msg & [e]] (throw (ex-info (format "Thaw failed: %s" msg) (let [v2+? (not v1-compatibility?)
ex (fn ex
([ msg] (ex nil msg))
([e msg] (throw (ex-info (format "Thaw failed: %s" msg)
{:opts (merge opts {:opts (merge opts
{:compressor compressor {:compressor compressor
:encryptor encryptor})} :encryptor encryptor})}
e))) e))))
thaw-data thaw-data
(fn [data-ba compressor-id encryptor-id] (fn [data-ba compressor-id encryptor-id ex-fn]
(let [compressor (if (identical? compressor :auto) (let [compressor (if (identical? compressor :auto)
(get-auto-compressor compressor-id) (get-auto-compressor compressor-id)
compressor) compressor)
@ -701,40 +712,62 @@
dis (DataInputStream. (ByteArrayInputStream. ba))] dis (DataInputStream. (ByteArrayInputStream. ba))]
(thaw-from-in! dis)) (thaw-from-in! dis))
(catch Exception e (catch Exception e (ex-fn e)))))
(ex "Decryption/decompression failure, or data unfrozen/damaged."
e)))))
;; This is hackish and can actually currently result in JVM core dumps ;; This is hackish and can actually currently result in JVM core dumps
;; due to buggy Snappy behaviour, Ref. http://goo.gl/mh7Rpy. ;; due to buggy Snappy behaviour, Ref. http://goo.gl/mh7Rpy
thaw-nippy-v1-data thaw-legacy-data
(fn [data-ba] (fn [data-ba ex-fn]
(if-not v1-compatibility? (thaw-data data-ba :snappy nil
(throw (Exception. "v1 compatibility disabled")) (fn [_] (thaw-data data-ba nil nil
(try (thaw-data data-ba :snappy nil) (fn [_] (ex-fn nil))))))]
(catch Exception _
(thaw-data data-ba nil nil)))))] (if no-header?
(if v2+?
(thaw-data ba :no-header :no-header
(fn [e] (ex e err-msg-unknown-thaw-failure)))
(thaw-data ba :no-header :no-header
(fn [e]
(thaw-legacy-data ba
(fn [_] (ex e err-msg-unknown-thaw-failure))))))
(if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?] (if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?]
:as head-meta}] (try-parse-header ba)] :as head-meta}] (try-parse-header ba)]
;; A well-formed header _appears_ to be present (it's possible though ;; A well-formed header _appears_ to be present (it's possible though
;; unlikely that this is a fluke and data is actually headerless): ;; unlikely that this is a fluke and data is actually headerless):
(try (thaw-data data-ba compressor-id encryptor-id) (if v2+?
(catch Exception e (thaw-data data-ba compressor-id encryptor-id
(try (thaw-nippy-v1-data data-ba) (fn [e]
(catch Exception _ (thaw-data ba :no-header :no-header
(fn [_]
(if unrecognized-meta? (if unrecognized-meta?
(ex "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?" (ex e err-msg-unrecognized-header)
e) (ex e err-msg-unknown-thaw-failure))))))
(throw e))))))
(thaw-data data-ba compressor-id encryptor-id
(fn [e]
(thaw-data ba :no-header :no-header
(fn [_]
(thaw-legacy-data ba
(fn [_]
(if unrecognized-meta?
(ex e err-msg-unrecognized-header)
(ex e err-msg-unknown-thaw-failure)))))))))
;; Well-formed header definitely not present ;; Well-formed header definitely not present
(try (thaw-nippy-v1-data ba) (if v2+?
(catch Exception _ (thaw-data ba :no-header :no-header
(thaw-data ba :no-header :no-header))))))) (fn [e] (ex e err-msg-unknown-thaw-failure)))
(comment (thaw (freeze "hello")) (thaw-data ba :no-header :no-header
(fn [e]
(thaw-legacy-data ba
(fn [_] (ex e err-msg-unknown-thaw-failure)))))))))))
(comment
(thaw (freeze "hello"))
(thaw (freeze "hello" {:compressor nil})) (thaw (freeze "hello" {:compressor nil}))
(thaw (freeze "hello" {:password [:salted "p"]})) ; ex: no pwd (thaw (freeze "hello" {:password [:salted "p"]})) ; ex: no pwd
(thaw (freeze "hello") {:password [:salted "p"]})) (thaw (freeze "hello") {:password [:salted "p"]}))
@ -922,7 +955,7 @@
;; Deprecated by :auto compressor selection ;; Deprecated by :auto compressor selection
(defrecord Compressable-LZMA2 [value]) ; Why was this `LZMA2` instead of `lzma2`? (defrecord Compressable-LZMA2 [value]) ; Why was this `LZMA2` instead of `lzma2`?
(extend-freeze Compressable-LZMA2 128 [x out] (extend-freeze Compressable-LZMA2 128 [x out]
(let [ba (freeze (:value x) {:skip-header? true :compressor nil}) (let [ba (freeze (:value x) {:no-header? true :compressor nil})
ba-len (alength ba) ba-len (alength ba)
compress? (> ba-len 1024)] compress? (> ba-len 1024)]
(.writeBoolean out compress?) (.writeBoolean out compress?)