luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add :auto legacy mode for _full_, transparent backwards-compatibility

Browse source
This commit is contained in:
Peter Taoussanis 2013-06-13 17:32:10 +07:00
parent d44dc44399
commit 15dd24ac06
5 changed files with 102 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version:
```clojure ```clojure
[com.taoensso/nippy "1.2.1"] ; Stable [com.taoensso/nippy "1.2.1"] ; Stable
[com.taoensso/nippy "2.0.0-alpha1"] ; Development (see notes below) [com.taoensso/nippy "2.0.0-alpha4"] ; Development (see notes below)
``` ```
2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`. 2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`.

2
project.clj
View file

@ -1,4 +1,4 @@
(defproject com.taoensso/nippy "2.0.0-alpha1" (defproject com.taoensso/nippy "2.0.0-alpha4"
:description "Clojure serialization library" :description "Clojure serialization library"
:url "https://github.com/ptaoussanis/nippy" :url "https://github.com/ptaoussanis/nippy"
:license {:name "Eclipse Public License" :license {:name "Eclipse Public License"

126
src/taoensso/nippy.clj
View file

@ -181,9 +181,9 @@
(utils/ba-concat header-ba data-ba))) (utils/ba-concat header-ba data-ba)))
(defn freeze (defn freeze
"Serializes arg (any Clojure data type) to a byte array. Enable "Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to
`:legacy-mode?` flag to produce bytes readable by Nippy < 2.x." true to produce bytes readble by Nippy < 2.x."
^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode?] ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode]
:or {print-dup? true :or {print-dup? true
compressor compression/default-snappy-compressor compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]] encryptor encryption/default-aes128-encryptor}}]]
@ -193,7 +193,7 @@
(let [ba (.toByteArray ba) (let [ba (.toByteArray ba)
ba (if compressor (compression/compress compressor ba) ba) ba (if compressor (compression/compress compressor ba) ba)
ba (if password (encryption/encrypt encryptor password ba) ba)] ba (if password (encryption/encrypt encryptor password ba) ba)]
(if legacy-mode? ba (wrap-nippy-header ba compressor encryptor password))))) (if legacy-mode ba (wrap-nippy-header ba compressor encryptor password)))))
;;;; Thawing ;;;; Thawing
@ -259,14 +259,24 @@
(throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) (throw (Exception. (str "Failed to thaw unknown type ID: " type-id))))))
(defn thaw (defn thaw
"Deserializes frozen bytes to their original Clojure data type. Enable "Deserializes frozen bytes to their original Clojure data type.
`:legacy-mode?` to read bytes written by Nippy < 2.x.
:legacy-mode can be set to one of the following values:
true - Read bytes as if written by Nippy < 2.x.
false - Read bytes as if written by Nippy >= 2.x.
:auto (default) - Try read bytes as if written by Nippy >= 2.x,
fall back to reading bytes as if written by Nippy < 2.x.
In most cases you'll want :auto if you're using a preexisting data set, and
`false` otherwise. Note that error message detail will be limited under the
:auto (default) mode.
WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless
you are sure you know what you're doing." you are sure you know what you're doing."
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode? [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode
strict?] strict?]
:or {compressor compression/default-snappy-compressor :or {legacy-mode :auto
compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]] encryptor encryption/default-aes128-encryptor}}]]
(let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed. " msg) e))) (let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed. " msg) e)))
@ -275,57 +285,73 @@
ba (if password (encryption/decrypt encryptor password ba) ba) ba (if password (encryption/decrypt encryptor password ba) ba)
ba (if compressor (compression/decompress compressor ba) ba) ba (if compressor (compression/decompress compressor ba) ba)
stream (DataInputStream. (ByteArrayInputStream. ba))] stream (DataInputStream. (ByteArrayInputStream. ba))]
(binding [*read-eval* read-eval?] (thaw-from-stream stream))))] (binding [*read-eval* read-eval?] (thaw-from-stream stream))))
(if legacy-mode? ; Nippy < 2.x maybe-headers
(try (thaw-data ba compressor password) (fn []
(catch Exception e (when-let [[[id-magic* & _ :as headers] data-ba] (utils/ba-split ba 5)]
(cond password (ex "Unencrypted data or wrong password?" e) (when (= id-magic* id-nippy-magic-prefix) ; Not a guarantee of correctness!
compressor (ex "Encrypted or uncompressed data?" e) [headers data-ba])))
:else (ex "Encrypted and/or compressed data?" e))))
;; Nippy >= 2.x, we have a header! legacy-thaw
(let [[[id-magic* id-header* id-comp* id-enc* _] data-ba] (fn [data-ba]
(utils/ba-split ba 5) (try (thaw-data data-ba compressor password)
(catch Exception e
(cond password (ex "Unencrypted data or wrong password?" e)
compressor (ex "Encrypted or uncompressed data?" e)
:else (ex "Encrypted and/or compressed data?" e)))))
compressed? (not (zero? id-comp*)) modern-thaw
encrypted? (not (zero? id-enc*))] (fn [data-ba compressed? encrypted?]
(try (thaw-data data-ba (when compressed? compressor)
(when encrypted? password))
(catch Exception e
(if (and encrypted? password)
(ex "Wrong password, or data may be corrupt?" e)
(ex "Data may be corrupt?" e)))))]
(cond (if (= legacy-mode true)
(not= id-magic* id-nippy-magic-prefix) (legacy-thaw ba) ; Read as legacy, and only as legacy
(ex (str "Not Nippy data, data frozen with Nippy < 2.x, " (if-let [[[_ id-header* id-comp* id-enc* _] data-ba] (maybe-headers)]
"or data may be corrupt?\n" (let [compressed? (not (zero? id-comp*))
"Enable `:legacy-mode?` option for data frozen with Nippy < 2.x.")) encrypted? (not (zero? id-enc*))]
(> id-header* id-nippy-header-ver) (if (= legacy-mode :auto)
(ex "Data frozen with newer Nippy version. Please upgrade.") (try ; Header looks okay: try read as modern, fall back to legacy
(modern-thaw data-ba compressed? encrypted?)
(catch Exception _ (legacy-thaw ba)))
(and strict? (not encrypted?) password) (cond ; Read as modern, and only as modern
(ex (str "Data is not encrypted. Try again w/o password.\n" (> id-header* id-nippy-header-ver)
"Disable `:strict?` option to ignore this error. ")) (ex "Data frozen with newer Nippy version. Please upgrade.")
(and strict? (not compressed?) compressor) (and strict? (not encrypted?) password)
(ex (str "Data is not compressed. Try again w/o compressor.\n" (ex (str "Data is not encrypted. Try again w/o password.\n"
"Disable `:strict?` option to ignore this error.")) "Disable `:strict?` option to ignore this error. "))
(and encrypted? (not password)) (and strict? (not compressed?) compressor)
(ex "Data is encrypted. Please try again with a password.") (ex (str "Data is not compressed. Try again w/o compressor.\n"
"Disable `:strict?` option to ignore this error."))
(and encrypted? password (and encrypted? (not password))
(not= id-enc* (encryption/header-id encryptor))) (ex "Data is encrypted. Please try again with a password.")
(ex "Data encrypted with a different Encrypter.")
(and compressed? compressor (and encrypted? password
(not= id-comp* (compression/header-id compressor))) (not= id-enc* (encryption/header-id encryptor)))
(ex "Data compressed with a different Compressor.") (ex "Data encrypted with a different Encrypter.")
:else (and compressed? compressor
(try (thaw-data data-ba (when compressed? compressor) (not= id-comp* (compression/header-id compressor)))
(when encrypted? password)) (ex "Data compressed with a different Compressor.")
(catch Exception e
(if (and encrypted? password) :else (modern-thaw data-ba compressed? encrypted?))))
(ex "Wrong password, or data may be corrupt?" e)
(ex "Data may be corrupt?" e))))))))) ;; Header definitely not okay
(if (= legacy-mode :auto)
(legacy-thaw ba)
(ex (str "Not Nippy data, data frozen with Nippy < 2.x, "
"or data may be corrupt?\n"
"See `:legacy-mode` option for data frozen with Nippy < 2.x.")))))))
(comment (thaw (freeze "hello")) (comment (thaw (freeze "hello"))
(thaw (freeze "hello" {:compressor nil})) (thaw (freeze "hello" {:compressor nil}))
@ -390,7 +416,7 @@
(freeze x {:print-dup? print-dup? (freeze x {:print-dup? print-dup?
:compressor (when compress? compression/default-snappy-compressor) :compressor (when compress? compression/default-snappy-compressor)
:password password :password password
:legacy-mode? true})) :legacy-mode true}))
(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead." (defn thaw-from-bytes "DEPRECATED: Use `thaw` instead."
[ba & {:keys [read-eval? compressed? password] [ba & {:keys [read-eval? compressed? password]
@ -398,4 +424,4 @@
(thaw ba {:read-eval? read-eval? (thaw ba {:read-eval? read-eval?
:compressor (when compressed? compression/default-snappy-compressor) :compressor (when compressed? compression/default-snappy-compressor)
:password password :password password
:legacy-mode? true})) :legacy-mode true}))

8
src/taoensso/nippy/utils.clj
View file

@ -78,9 +78,11 @@
out)) out))
(defn ba-split [^bytes ba ^Integer idx] (defn ba-split [^bytes ba ^Integer idx]
[(java.util.Arrays/copyOfRange ba 0 idx) (let [s (alength ba)]
(java.util.Arrays/copyOfRange ba idx (alength ba))]) (when (> s idx)
[(java.util.Arrays/copyOfRange ba 0 idx)
(java.util.Arrays/copyOfRange ba idx s)])))
(comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar"))) (comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar")))
(let [[x y] (ba-split (.getBytes "foobar") 3)] (let [[x y] (ba-split (.getBytes "foobar") 5)]
[(String. x) (String. y)])) [(String. x) (String. y)]))

39
test/taoensso/nippy/tests/main.clj
View file

@ -6,21 +6,14 @@
;; Remove stuff from stress-data that breaks roundtrip equality ;; Remove stuff from stress-data that breaks roundtrip equality
(def test-data (dissoc nippy/stress-data :bytes)) (def test-data (dissoc nippy/stress-data :bytes))
(def roundtrip-defaults (comp thaw freeze)) ;;;; Basic data integrity
(def roundtrip-encrypted (comp #(thaw % {:password [:salted "p"]}) (expect test-data ((comp thaw freeze) test-data))
#(freeze % {:password [:salted "p"]}))) (expect test-data ((comp #(thaw % {:legacy-mode :auto})
(def roundtrip-defaults-legacy (comp #(thaw % {:legacy-mode? true}) #(freeze % {:legacy-mode true}))
#(freeze % {:legacy-mode? true}))) test-data))
(def roundtrip-encrypted-legacy (comp #(thaw % {:password [:salted "p"] (expect test-data ((comp #(thaw % {:password [:salted "p"]})
:legacy-mode? true}) #(freeze % {:password [:salted "p"]}))
#(freeze % {:password [:salted "p"] test-data))
:legacy-mode? true})))
;;; Basic data integrity
(expect test-data (roundtrip-defaults test-data))
(expect test-data (roundtrip-encrypted test-data))
(expect test-data (roundtrip-defaults-legacy test-data))
(expect test-data (roundtrip-encrypted-legacy test-data))
(expect ; Snappy lib compatibility (for legacy versions of Nippy) (expect ; Snappy lib compatibility (for legacy versions of Nippy)
(let [^bytes raw-ba (freeze test-data {:compressor nil}) (let [^bytes raw-ba (freeze test-data {:compressor nil})
@ -32,17 +25,21 @@
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
;;; API stuff ;;;; API stuff
;; Strict/auto mode - compression ;;; Strict/auto mode - compression
(expect test-data (thaw (freeze test-data {:compressor nil}))) (expect test-data (thaw (freeze test-data {:compressor nil})))
(expect Exception (thaw (freeze test-data {:compressor nil}) {:strict? true})) (expect Exception (thaw (freeze test-data {:compressor nil})
{:legacy-mode false
:strict? true}))
;; Strict/auto mode - encryption ;;; Strict/auto mode - encryption
(expect test-data (thaw (freeze test-data) {:password [:salted "p"]})) (expect test-data (thaw (freeze test-data) {:password [:salted "p"]}))
(expect Exception (thaw (freeze test-data) {:password [:salted "p"] :strict? true})) (expect Exception (thaw (freeze test-data) {:password [:salted "p"]
:legacy-mode false
:strict? true}))
;; Encryption - passwords ;;; Encryption - passwords
(expect Exception (thaw (freeze test-data {:password "malformed"}))) (expect Exception (thaw (freeze test-data {:password "malformed"})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) (expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
(expect test-data (thaw (freeze test-data {:password [:salted "p"]}) (expect test-data (thaw (freeze test-data {:password [:salted "p"]})