Add :auto legacy mode for _full_, transparent backwards-compatibility
This commit is contained in:
Peter Taoussanis
parent
d44dc44399
commit
15dd24ac06
5 changed files with 102 additions and 77 deletions
|
|
@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version:
|
|||
|
||||
```clojure
|
||||
[com.taoensso/nippy "1.2.1"] ; Stable
|
||||
[com.taoensso/nippy "2.0.0-alpha1"] ; Development (see notes below)
|
||||
[com.taoensso/nippy "2.0.0-alpha4"] ; Development (see notes below)
|
||||
```
|
||||
|
||||
2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
(defproject com.taoensso/nippy "2.0.0-alpha1"
|
||||
(defproject com.taoensso/nippy "2.0.0-alpha4"
|
||||
:description "Clojure serialization library"
|
||||
:url "https://github.com/ptaoussanis/nippy"
|
||||
:license {:name "Eclipse Public License"
|
||||
|
|
@ -20,4 +20,4 @@
|
|||
[lein-autoexpect "0.2.5"]
|
||||
[codox "0.6.4"]]
|
||||
:min-lein-version "2.0.0"
|
||||
:warn-on-reflection true)
|
||||
:warn-on-reflection true)
|
||||
|
|
@ -181,9 +181,9 @@
|
|||
(utils/ba-concat header-ba data-ba)))
|
||||
|
||||
(defn freeze
|
||||
"Serializes arg (any Clojure data type) to a byte array. Enable
|
||||
`:legacy-mode?` flag to produce bytes readable by Nippy < 2.x."
|
||||
^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode?]
|
||||
"Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to
|
||||
true to produce bytes readble by Nippy < 2.x."
|
||||
^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode]
|
||||
:or {print-dup? true
|
||||
compressor compression/default-snappy-compressor
|
||||
encryptor encryption/default-aes128-encryptor}}]]
|
||||
|
|
@ -193,7 +193,7 @@
|
|||
(let [ba (.toByteArray ba)
|
||||
ba (if compressor (compression/compress compressor ba) ba)
|
||||
ba (if password (encryption/encrypt encryptor password ba) ba)]
|
||||
(if legacy-mode? ba (wrap-nippy-header ba compressor encryptor password)))))
|
||||
(if legacy-mode ba (wrap-nippy-header ba compressor encryptor password)))))
|
||||
|
||||
;;;; Thawing
|
||||
|
||||
|
|
@ -259,14 +259,24 @@
|
|||
(throw (Exception. (str "Failed to thaw unknown type ID: " type-id))))))
|
||||
|
||||
(defn thaw
|
||||
"Deserializes frozen bytes to their original Clojure data type. Enable
|
||||
`:legacy-mode?` to read bytes written by Nippy < 2.x.
|
||||
"Deserializes frozen bytes to their original Clojure data type.
|
||||
|
||||
:legacy-mode can be set to one of the following values:
|
||||
true - Read bytes as if written by Nippy < 2.x.
|
||||
false - Read bytes as if written by Nippy >= 2.x.
|
||||
:auto (default) - Try read bytes as if written by Nippy >= 2.x,
|
||||
fall back to reading bytes as if written by Nippy < 2.x.
|
||||
|
||||
In most cases you'll want :auto if you're using a preexisting data set, and
|
||||
`false` otherwise. Note that error message detail will be limited under the
|
||||
:auto (default) mode.
|
||||
|
||||
WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless
|
||||
you are sure you know what you're doing."
|
||||
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode?
|
||||
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode
|
||||
strict?]
|
||||
:or {compressor compression/default-snappy-compressor
|
||||
:or {legacy-mode :auto
|
||||
compressor compression/default-snappy-compressor
|
||||
encryptor encryption/default-aes128-encryptor}}]]
|
||||
|
||||
(let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed. " msg) e)))
|
||||
|
|
@ -275,57 +285,73 @@
|
|||
ba (if password (encryption/decrypt encryptor password ba) ba)
|
||||
ba (if compressor (compression/decompress compressor ba) ba)
|
||||
stream (DataInputStream. (ByteArrayInputStream. ba))]
|
||||
(binding [*read-eval* read-eval?] (thaw-from-stream stream))))]
|
||||
(binding [*read-eval* read-eval?] (thaw-from-stream stream))))
|
||||
|
||||
(if legacy-mode? ; Nippy < 2.x
|
||||
(try (thaw-data ba compressor password)
|
||||
(catch Exception e
|
||||
(cond password (ex "Unencrypted data or wrong password?" e)
|
||||
compressor (ex "Encrypted or uncompressed data?" e)
|
||||
:else (ex "Encrypted and/or compressed data?" e))))
|
||||
maybe-headers
|
||||
(fn []
|
||||
(when-let [[[id-magic* & _ :as headers] data-ba] (utils/ba-split ba 5)]
|
||||
(when (= id-magic* id-nippy-magic-prefix) ; Not a guarantee of correctness!
|
||||
[headers data-ba])))
|
||||
|
||||
;; Nippy >= 2.x, we have a header!
|
||||
(let [[[id-magic* id-header* id-comp* id-enc* _] data-ba]
|
||||
(utils/ba-split ba 5)
|
||||
legacy-thaw
|
||||
(fn [data-ba]
|
||||
(try (thaw-data data-ba compressor password)
|
||||
(catch Exception e
|
||||
(cond password (ex "Unencrypted data or wrong password?" e)
|
||||
compressor (ex "Encrypted or uncompressed data?" e)
|
||||
:else (ex "Encrypted and/or compressed data?" e)))))
|
||||
|
||||
compressed? (not (zero? id-comp*))
|
||||
encrypted? (not (zero? id-enc*))]
|
||||
modern-thaw
|
||||
(fn [data-ba compressed? encrypted?]
|
||||
(try (thaw-data data-ba (when compressed? compressor)
|
||||
(when encrypted? password))
|
||||
(catch Exception e
|
||||
(if (and encrypted? password)
|
||||
(ex "Wrong password, or data may be corrupt?" e)
|
||||
(ex "Data may be corrupt?" e)))))]
|
||||
|
||||
(cond
|
||||
(not= id-magic* id-nippy-magic-prefix)
|
||||
(ex (str "Not Nippy data, data frozen with Nippy < 2.x, "
|
||||
"or data may be corrupt?\n"
|
||||
"Enable `:legacy-mode?` option for data frozen with Nippy < 2.x."))
|
||||
(if (= legacy-mode true)
|
||||
(legacy-thaw ba) ; Read as legacy, and only as legacy
|
||||
(if-let [[[_ id-header* id-comp* id-enc* _] data-ba] (maybe-headers)]
|
||||
(let [compressed? (not (zero? id-comp*))
|
||||
encrypted? (not (zero? id-enc*))]
|
||||
|
||||
(> id-header* id-nippy-header-ver)
|
||||
(ex "Data frozen with newer Nippy version. Please upgrade.")
|
||||
(if (= legacy-mode :auto)
|
||||
(try ; Header looks okay: try read as modern, fall back to legacy
|
||||
(modern-thaw data-ba compressed? encrypted?)
|
||||
(catch Exception _ (legacy-thaw ba)))
|
||||
|
||||
(and strict? (not encrypted?) password)
|
||||
(ex (str "Data is not encrypted. Try again w/o password.\n"
|
||||
"Disable `:strict?` option to ignore this error. "))
|
||||
(cond ; Read as modern, and only as modern
|
||||
(> id-header* id-nippy-header-ver)
|
||||
(ex "Data frozen with newer Nippy version. Please upgrade.")
|
||||
|
||||
(and strict? (not compressed?) compressor)
|
||||
(ex (str "Data is not compressed. Try again w/o compressor.\n"
|
||||
"Disable `:strict?` option to ignore this error."))
|
||||
(and strict? (not encrypted?) password)
|
||||
(ex (str "Data is not encrypted. Try again w/o password.\n"
|
||||
"Disable `:strict?` option to ignore this error. "))
|
||||
|
||||
(and encrypted? (not password))
|
||||
(ex "Data is encrypted. Please try again with a password.")
|
||||
(and strict? (not compressed?) compressor)
|
||||
(ex (str "Data is not compressed. Try again w/o compressor.\n"
|
||||
"Disable `:strict?` option to ignore this error."))
|
||||
|
||||
(and encrypted? password
|
||||
(not= id-enc* (encryption/header-id encryptor)))
|
||||
(ex "Data encrypted with a different Encrypter.")
|
||||
(and encrypted? (not password))
|
||||
(ex "Data is encrypted. Please try again with a password.")
|
||||
|
||||
(and compressed? compressor
|
||||
(not= id-comp* (compression/header-id compressor)))
|
||||
(ex "Data compressed with a different Compressor.")
|
||||
(and encrypted? password
|
||||
(not= id-enc* (encryption/header-id encryptor)))
|
||||
(ex "Data encrypted with a different Encrypter.")
|
||||
|
||||
:else
|
||||
(try (thaw-data data-ba (when compressed? compressor)
|
||||
(when encrypted? password))
|
||||
(catch Exception e
|
||||
(if (and encrypted? password)
|
||||
(ex "Wrong password, or data may be corrupt?" e)
|
||||
(ex "Data may be corrupt?" e)))))))))
|
||||
(and compressed? compressor
|
||||
(not= id-comp* (compression/header-id compressor)))
|
||||
(ex "Data compressed with a different Compressor.")
|
||||
|
||||
:else (modern-thaw data-ba compressed? encrypted?))))
|
||||
|
||||
;; Header definitely not okay
|
||||
(if (= legacy-mode :auto)
|
||||
(legacy-thaw ba)
|
||||
(ex (str "Not Nippy data, data frozen with Nippy < 2.x, "
|
||||
"or data may be corrupt?\n"
|
||||
"See `:legacy-mode` option for data frozen with Nippy < 2.x.")))))))
|
||||
|
||||
(comment (thaw (freeze "hello"))
|
||||
(thaw (freeze "hello" {:compressor nil}))
|
||||
|
|
@ -390,7 +416,7 @@
|
|||
(freeze x {:print-dup? print-dup?
|
||||
:compressor (when compress? compression/default-snappy-compressor)
|
||||
:password password
|
||||
:legacy-mode? true}))
|
||||
:legacy-mode true}))
|
||||
|
||||
(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead."
|
||||
[ba & {:keys [read-eval? compressed? password]
|
||||
|
|
@ -398,4 +424,4 @@
|
|||
(thaw ba {:read-eval? read-eval?
|
||||
:compressor (when compressed? compression/default-snappy-compressor)
|
||||
:password password
|
||||
:legacy-mode? true}))
|
||||
:legacy-mode true}))
|
||||
|
|
@ -78,9 +78,11 @@
|
|||
out))
|
||||
|
||||
(defn ba-split [^bytes ba ^Integer idx]
|
||||
[(java.util.Arrays/copyOfRange ba 0 idx)
|
||||
(java.util.Arrays/copyOfRange ba idx (alength ba))])
|
||||
(let [s (alength ba)]
|
||||
(when (> s idx)
|
||||
[(java.util.Arrays/copyOfRange ba 0 idx)
|
||||
(java.util.Arrays/copyOfRange ba idx s)])))
|
||||
|
||||
(comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar")))
|
||||
(let [[x y] (ba-split (.getBytes "foobar") 3)]
|
||||
(let [[x y] (ba-split (.getBytes "foobar") 5)]
|
||||
[(String. x) (String. y)]))
|
||||
|
|
@ -6,21 +6,14 @@
|
|||
;; Remove stuff from stress-data that breaks roundtrip equality
|
||||
(def test-data (dissoc nippy/stress-data :bytes))
|
||||
|
||||
(def roundtrip-defaults (comp thaw freeze))
|
||||
(def roundtrip-encrypted (comp #(thaw % {:password [:salted "p"]})
|
||||
#(freeze % {:password [:salted "p"]})))
|
||||
(def roundtrip-defaults-legacy (comp #(thaw % {:legacy-mode? true})
|
||||
#(freeze % {:legacy-mode? true})))
|
||||
(def roundtrip-encrypted-legacy (comp #(thaw % {:password [:salted "p"]
|
||||
:legacy-mode? true})
|
||||
#(freeze % {:password [:salted "p"]
|
||||
:legacy-mode? true})))
|
||||
|
||||
;;; Basic data integrity
|
||||
(expect test-data (roundtrip-defaults test-data))
|
||||
(expect test-data (roundtrip-encrypted test-data))
|
||||
(expect test-data (roundtrip-defaults-legacy test-data))
|
||||
(expect test-data (roundtrip-encrypted-legacy test-data))
|
||||
;;;; Basic data integrity
|
||||
(expect test-data ((comp thaw freeze) test-data))
|
||||
(expect test-data ((comp #(thaw % {:legacy-mode :auto})
|
||||
#(freeze % {:legacy-mode true}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:password [:salted "p"]})
|
||||
#(freeze % {:password [:salted "p"]}))
|
||||
test-data))
|
||||
|
||||
(expect ; Snappy lib compatibility (for legacy versions of Nippy)
|
||||
(let [^bytes raw-ba (freeze test-data {:compressor nil})
|
||||
|
|
@ -32,17 +25,21 @@
|
|||
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
|
||||
|
||||
;;; API stuff
|
||||
;;;; API stuff
|
||||
|
||||
;; Strict/auto mode - compression
|
||||
;;; Strict/auto mode - compression
|
||||
(expect test-data (thaw (freeze test-data {:compressor nil})))
|
||||
(expect Exception (thaw (freeze test-data {:compressor nil}) {:strict? true}))
|
||||
(expect Exception (thaw (freeze test-data {:compressor nil})
|
||||
{:legacy-mode false
|
||||
:strict? true}))
|
||||
|
||||
;; Strict/auto mode - encryption
|
||||
;;; Strict/auto mode - encryption
|
||||
(expect test-data (thaw (freeze test-data) {:password [:salted "p"]}))
|
||||
(expect Exception (thaw (freeze test-data) {:password [:salted "p"] :strict? true}))
|
||||
(expect Exception (thaw (freeze test-data) {:password [:salted "p"]
|
||||
:legacy-mode false
|
||||
:strict? true}))
|
||||
|
||||
;; Encryption - passwords
|
||||
;;; Encryption - passwords
|
||||
(expect Exception (thaw (freeze test-data {:password "malformed"})))
|
||||
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
|
||||
(expect test-data (thaw (freeze test-data {:password [:salted "p"]})
|
||||
|
|
|
|||
Loading…
Reference in a new issue