diff --git a/README.md b/README.md index c1d6dc4..f6f2903 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version: ```clojure [com.taoensso/nippy "1.2.1"] ; Stable -[com.taoensso/nippy "2.0.0-alpha1"] ; Development (see notes below) +[com.taoensso/nippy "2.0.0-alpha4"] ; Development (see notes below) ``` 2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`. diff --git a/project.clj b/project.clj index 7f34a34..2fa8273 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.0.0-alpha1" +(defproject com.taoensso/nippy "2.0.0-alpha4" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License" @@ -20,4 +20,4 @@ [lein-autoexpect "0.2.5"] [codox "0.6.4"]] :min-lein-version "2.0.0" - :warn-on-reflection true) + :warn-on-reflection true) \ No newline at end of file diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index ba54b48..c9c8204 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -181,9 +181,9 @@ (utils/ba-concat header-ba data-ba))) (defn freeze - "Serializes arg (any Clojure data type) to a byte array. Enable - `:legacy-mode?` flag to produce bytes readable by Nippy < 2.x." - ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode?] + "Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to + true to produce bytes readble by Nippy < 2.x." + ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode] :or {print-dup? true compressor compression/default-snappy-compressor encryptor encryption/default-aes128-encryptor}}]] @@ -193,7 +193,7 @@ (let [ba (.toByteArray ba) ba (if compressor (compression/compress compressor ba) ba) ba (if password (encryption/encrypt encryptor password ba) ba)] - (if legacy-mode? ba (wrap-nippy-header ba compressor encryptor password))))) + (if legacy-mode ba (wrap-nippy-header ba compressor encryptor password))))) ;;;; Thawing @@ -259,14 +259,24 @@ (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) (defn thaw - "Deserializes frozen bytes to their original Clojure data type. Enable - `:legacy-mode?` to read bytes written by Nippy < 2.x. + "Deserializes frozen bytes to their original Clojure data type. + + :legacy-mode can be set to one of the following values: + true - Read bytes as if written by Nippy < 2.x. + false - Read bytes as if written by Nippy >= 2.x. + :auto (default) - Try read bytes as if written by Nippy >= 2.x, + fall back to reading bytes as if written by Nippy < 2.x. + + In most cases you'll want :auto if you're using a preexisting data set, and + `false` otherwise. Note that error message detail will be limited under the + :auto (default) mode. WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless you are sure you know what you're doing." - [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode? + [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode strict?] - :or {compressor compression/default-snappy-compressor + :or {legacy-mode :auto + compressor compression/default-snappy-compressor encryptor encryption/default-aes128-encryptor}}]] (let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed. " msg) e))) @@ -275,57 +285,73 @@ ba (if password (encryption/decrypt encryptor password ba) ba) ba (if compressor (compression/decompress compressor ba) ba) stream (DataInputStream. (ByteArrayInputStream. ba))] - (binding [*read-eval* read-eval?] (thaw-from-stream stream))))] + (binding [*read-eval* read-eval?] (thaw-from-stream stream)))) - (if legacy-mode? ; Nippy < 2.x - (try (thaw-data ba compressor password) - (catch Exception e - (cond password (ex "Unencrypted data or wrong password?" e) - compressor (ex "Encrypted or uncompressed data?" e) - :else (ex "Encrypted and/or compressed data?" e)))) + maybe-headers + (fn [] + (when-let [[[id-magic* & _ :as headers] data-ba] (utils/ba-split ba 5)] + (when (= id-magic* id-nippy-magic-prefix) ; Not a guarantee of correctness! + [headers data-ba]))) - ;; Nippy >= 2.x, we have a header! - (let [[[id-magic* id-header* id-comp* id-enc* _] data-ba] - (utils/ba-split ba 5) + legacy-thaw + (fn [data-ba] + (try (thaw-data data-ba compressor password) + (catch Exception e + (cond password (ex "Unencrypted data or wrong password?" e) + compressor (ex "Encrypted or uncompressed data?" e) + :else (ex "Encrypted and/or compressed data?" e))))) - compressed? (not (zero? id-comp*)) - encrypted? (not (zero? id-enc*))] + modern-thaw + (fn [data-ba compressed? encrypted?] + (try (thaw-data data-ba (when compressed? compressor) + (when encrypted? password)) + (catch Exception e + (if (and encrypted? password) + (ex "Wrong password, or data may be corrupt?" e) + (ex "Data may be corrupt?" e)))))] - (cond - (not= id-magic* id-nippy-magic-prefix) - (ex (str "Not Nippy data, data frozen with Nippy < 2.x, " - "or data may be corrupt?\n" - "Enable `:legacy-mode?` option for data frozen with Nippy < 2.x.")) + (if (= legacy-mode true) + (legacy-thaw ba) ; Read as legacy, and only as legacy + (if-let [[[_ id-header* id-comp* id-enc* _] data-ba] (maybe-headers)] + (let [compressed? (not (zero? id-comp*)) + encrypted? (not (zero? id-enc*))] - (> id-header* id-nippy-header-ver) - (ex "Data frozen with newer Nippy version. Please upgrade.") + (if (= legacy-mode :auto) + (try ; Header looks okay: try read as modern, fall back to legacy + (modern-thaw data-ba compressed? encrypted?) + (catch Exception _ (legacy-thaw ba))) - (and strict? (not encrypted?) password) - (ex (str "Data is not encrypted. Try again w/o password.\n" - "Disable `:strict?` option to ignore this error. ")) + (cond ; Read as modern, and only as modern + (> id-header* id-nippy-header-ver) + (ex "Data frozen with newer Nippy version. Please upgrade.") - (and strict? (not compressed?) compressor) - (ex (str "Data is not compressed. Try again w/o compressor.\n" - "Disable `:strict?` option to ignore this error.")) + (and strict? (not encrypted?) password) + (ex (str "Data is not encrypted. Try again w/o password.\n" + "Disable `:strict?` option to ignore this error. ")) - (and encrypted? (not password)) - (ex "Data is encrypted. Please try again with a password.") + (and strict? (not compressed?) compressor) + (ex (str "Data is not compressed. Try again w/o compressor.\n" + "Disable `:strict?` option to ignore this error.")) - (and encrypted? password - (not= id-enc* (encryption/header-id encryptor))) - (ex "Data encrypted with a different Encrypter.") + (and encrypted? (not password)) + (ex "Data is encrypted. Please try again with a password.") - (and compressed? compressor - (not= id-comp* (compression/header-id compressor))) - (ex "Data compressed with a different Compressor.") + (and encrypted? password + (not= id-enc* (encryption/header-id encryptor))) + (ex "Data encrypted with a different Encrypter.") - :else - (try (thaw-data data-ba (when compressed? compressor) - (when encrypted? password)) - (catch Exception e - (if (and encrypted? password) - (ex "Wrong password, or data may be corrupt?" e) - (ex "Data may be corrupt?" e))))))))) + (and compressed? compressor + (not= id-comp* (compression/header-id compressor))) + (ex "Data compressed with a different Compressor.") + + :else (modern-thaw data-ba compressed? encrypted?)))) + + ;; Header definitely not okay + (if (= legacy-mode :auto) + (legacy-thaw ba) + (ex (str "Not Nippy data, data frozen with Nippy < 2.x, " + "or data may be corrupt?\n" + "See `:legacy-mode` option for data frozen with Nippy < 2.x."))))))) (comment (thaw (freeze "hello")) (thaw (freeze "hello" {:compressor nil})) @@ -390,7 +416,7 @@ (freeze x {:print-dup? print-dup? :compressor (when compress? compression/default-snappy-compressor) :password password - :legacy-mode? true})) + :legacy-mode true})) (defn thaw-from-bytes "DEPRECATED: Use `thaw` instead." [ba & {:keys [read-eval? compressed? password] @@ -398,4 +424,4 @@ (thaw ba {:read-eval? read-eval? :compressor (when compressed? compression/default-snappy-compressor) :password password - :legacy-mode? true})) \ No newline at end of file + :legacy-mode true})) \ No newline at end of file diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index d4d1910..787b707 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -78,9 +78,11 @@ out)) (defn ba-split [^bytes ba ^Integer idx] - [(java.util.Arrays/copyOfRange ba 0 idx) - (java.util.Arrays/copyOfRange ba idx (alength ba))]) + (let [s (alength ba)] + (when (> s idx) + [(java.util.Arrays/copyOfRange ba 0 idx) + (java.util.Arrays/copyOfRange ba idx s)]))) (comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar"))) - (let [[x y] (ba-split (.getBytes "foobar") 3)] + (let [[x y] (ba-split (.getBytes "foobar") 5)] [(String. x) (String. y)])) \ No newline at end of file diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index a0d4e07..85a02f3 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -6,21 +6,14 @@ ;; Remove stuff from stress-data that breaks roundtrip equality (def test-data (dissoc nippy/stress-data :bytes)) -(def roundtrip-defaults (comp thaw freeze)) -(def roundtrip-encrypted (comp #(thaw % {:password [:salted "p"]}) - #(freeze % {:password [:salted "p"]}))) -(def roundtrip-defaults-legacy (comp #(thaw % {:legacy-mode? true}) - #(freeze % {:legacy-mode? true}))) -(def roundtrip-encrypted-legacy (comp #(thaw % {:password [:salted "p"] - :legacy-mode? true}) - #(freeze % {:password [:salted "p"] - :legacy-mode? true}))) - -;;; Basic data integrity -(expect test-data (roundtrip-defaults test-data)) -(expect test-data (roundtrip-encrypted test-data)) -(expect test-data (roundtrip-defaults-legacy test-data)) -(expect test-data (roundtrip-encrypted-legacy test-data)) +;;;; Basic data integrity +(expect test-data ((comp thaw freeze) test-data)) +(expect test-data ((comp #(thaw % {:legacy-mode :auto}) + #(freeze % {:legacy-mode true})) + test-data)) +(expect test-data ((comp #(thaw % {:password [:salted "p"]}) + #(freeze % {:password [:salted "p"]})) + test-data)) (expect ; Snappy lib compatibility (for legacy versions of Nippy) (let [^bytes raw-ba (freeze test-data {:compressor nil}) @@ -32,17 +25,21 @@ (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) -;;; API stuff +;;;; API stuff -;; Strict/auto mode - compression +;;; Strict/auto mode - compression (expect test-data (thaw (freeze test-data {:compressor nil}))) -(expect Exception (thaw (freeze test-data {:compressor nil}) {:strict? true})) +(expect Exception (thaw (freeze test-data {:compressor nil}) + {:legacy-mode false + :strict? true})) -;; Strict/auto mode - encryption +;;; Strict/auto mode - encryption (expect test-data (thaw (freeze test-data) {:password [:salted "p"]})) -(expect Exception (thaw (freeze test-data) {:password [:salted "p"] :strict? true})) +(expect Exception (thaw (freeze test-data) {:password [:salted "p"] + :legacy-mode false + :strict? true})) -;; Encryption - passwords +;;; Encryption - passwords (expect Exception (thaw (freeze test-data {:password "malformed"}))) (expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) (expect test-data (thaw (freeze test-data {:password [:salted "p"]})