diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 7324f2a..ba54b48 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -1,16 +1,28 @@ (ns taoensso.nippy - "Simple, high-performance Clojure serialization library. Adapted from - Deep-Freeze." + "Simple, high-performance Clojure serialization library. Originally adapted + from Deep-Freeze." {:author "Peter Taoussanis"} - (:require [taoensso.nippy.utils :as utils] - [taoensso.nippy.crypto :as crypto]) + (:require [taoensso.nippy + (utils :as utils) + (compression :as compression) + (encryption :as encryption)]) (:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream ByteArrayInputStream] [clojure.lang Keyword BigInt Ratio PersistentQueue PersistentTreeMap PersistentTreeSet IPersistentList IPersistentVector IPersistentMap IPersistentSet IPersistentCollection])) -;;;; Header IDs ; TODO +;; TODO Allow ba or wrapped-ba input? +;; TODO Provide ToFreeze, Frozen, Encrypted, etc. tooling helpers + +;;;; Header IDs +;; Nippy 2.x+ prefixes frozen data with a 5-byte header: + +(def ^:const id-nippy-magic-prefix (byte 17)) +(def ^:const id-nippy-header-ver (byte 0)) +;; * Compressor id (0 if no compressor) +;; * Encryptor id (0 if no encryptor) +(def ^:const id-nippy-reserved (byte 0)) ;;;; Data type IDs @@ -159,7 +171,29 @@ ;; Use Clojure's own reader as final fallback (freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8"))) -;; TODO New `freeze` API +(defn- wrap-nippy-header [data-ba compressor encryptor password] + (let [header-ba (byte-array + [id-nippy-magic-prefix + id-nippy-header-ver + (byte (if compressor (compression/header-id compressor) 0)) + (byte (if password (encryption/header-id encryptor) 0)) + id-nippy-reserved])] + (utils/ba-concat header-ba data-ba))) + +(defn freeze + "Serializes arg (any Clojure data type) to a byte array. Enable + `:legacy-mode?` flag to produce bytes readable by Nippy < 2.x." + ^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode?] + :or {print-dup? true + compressor compression/default-snappy-compressor + encryptor encryption/default-aes128-encryptor}}]] + (let [ba (ByteArrayOutputStream.) + stream (DataOutputStream. ba)] + (binding [*print-dup* print-dup?] (freeze-to-stream x stream)) + (let [ba (.toByteArray ba) + ba (if compressor (compression/compress compressor ba) ba) + ba (if password (encryption/encrypt encryptor password ba) ba)] + (if legacy-mode? ba (wrap-nippy-header ba compressor encryptor password))))) ;;;; Thawing @@ -224,7 +258,80 @@ (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) -;; TODO New `thaw` API +(defn thaw + "Deserializes frozen bytes to their original Clojure data type. Enable + `:legacy-mode?` to read bytes written by Nippy < 2.x. + + WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless + you are sure you know what you're doing." + [^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode? + strict?] + :or {compressor compression/default-snappy-compressor + encryptor encryption/default-aes128-encryptor}}]] + + (let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed. " msg) e))) + thaw-data (fn [data-ba compressor password] + (let [ba data-ba + ba (if password (encryption/decrypt encryptor password ba) ba) + ba (if compressor (compression/decompress compressor ba) ba) + stream (DataInputStream. (ByteArrayInputStream. ba))] + (binding [*read-eval* read-eval?] (thaw-from-stream stream))))] + + (if legacy-mode? ; Nippy < 2.x + (try (thaw-data ba compressor password) + (catch Exception e + (cond password (ex "Unencrypted data or wrong password?" e) + compressor (ex "Encrypted or uncompressed data?" e) + :else (ex "Encrypted and/or compressed data?" e)))) + + ;; Nippy >= 2.x, we have a header! + (let [[[id-magic* id-header* id-comp* id-enc* _] data-ba] + (utils/ba-split ba 5) + + compressed? (not (zero? id-comp*)) + encrypted? (not (zero? id-enc*))] + + (cond + (not= id-magic* id-nippy-magic-prefix) + (ex (str "Not Nippy data, data frozen with Nippy < 2.x, " + "or data may be corrupt?\n" + "Enable `:legacy-mode?` option for data frozen with Nippy < 2.x.")) + + (> id-header* id-nippy-header-ver) + (ex "Data frozen with newer Nippy version. Please upgrade.") + + (and strict? (not encrypted?) password) + (ex (str "Data is not encrypted. Try again w/o password.\n" + "Disable `:strict?` option to ignore this error. ")) + + (and strict? (not compressed?) compressor) + (ex (str "Data is not compressed. Try again w/o compressor.\n" + "Disable `:strict?` option to ignore this error.")) + + (and encrypted? (not password)) + (ex "Data is encrypted. Please try again with a password.") + + (and encrypted? password + (not= id-enc* (encryption/header-id encryptor))) + (ex "Data encrypted with a different Encrypter.") + + (and compressed? compressor + (not= id-comp* (compression/header-id compressor))) + (ex "Data compressed with a different Compressor.") + + :else + (try (thaw-data data-ba (when compressed? compressor) + (when encrypted? password)) + (catch Exception e + (if (and encrypted? password) + (ex "Wrong password, or data may be corrupt?" e) + (ex "Data may be corrupt?" e))))))))) + +(comment (thaw (freeze "hello")) + (thaw (freeze "hello" {:compressor nil})) + (thaw (freeze "hello" {:compressor nil}) {:strict? true}) ; ex + (thaw (freeze "hello" {:password [:salted "p"]})) ; ex + (thaw (freeze "hello") {:password [:salted "p"]})) ;;;; Stress data @@ -276,39 +383,19 @@ ;;;; Deprecated API -;; TODO Rewrite in :legacy terms (defn freeze-to-bytes "DEPRECATED: Use `freeze` instead." - ^bytes [x & {:keys [compress? print-dup? password] - :or {compress? true - print-dup? true}}] - (let [ba (ByteArrayOutputStream.) - stream (DataOutputStream. ba)] - (binding [*print-dup* print-dup?] (freeze-to-stream x stream)) - (let [ba (.toByteArray ba) - ba (if compress? (utils/compress-snappy ba) ba) - ba (if password (crypto/encrypt-aes128 password ba) ba)] - ba))) + ^bytes [x & {:keys [print-dup? compress? password] + :or {print-dup? true + compress? true}}] + (freeze x {:print-dup? print-dup? + :compressor (when compress? compression/default-snappy-compressor) + :password password + :legacy-mode? true})) -;; TODO Rewrite in :legacy terms (defn thaw-from-bytes "DEPRECATED: Use `thaw` instead." - [ba & {:keys [compressed? read-eval? password] - :or {read-eval? false ; For `read-string` injection safety - NB!!! - compressed? true}}] - (try - (let [ba (if password (crypto/decrypt-aes128 password ba) ba) - ba (if compressed? (utils/uncompress-snappy ba) ba) - stream (DataInputStream. (ByteArrayInputStream. ba))] - (binding [*read-eval* read-eval?] (thaw-from-stream stream))) - (catch Exception e - (throw (Exception. - (cond password "Thaw failed. Unencrypted data or bad password?" - compressed? "Thaw failed. Encrypted or uncompressed data?" - :else "Thaw failed. Encrypted and/or compressed data?") - e))))) - -(comment - ;; Errors - (-> (freeze-to-bytes "my data" :password [:salted "password"]) - (thaw-from-bytes)) - (-> (freeze-to-bytes "my data" :compress? true) - (thaw-from-bytes :compressed? false))) \ No newline at end of file + [ba & {:keys [read-eval? compressed? password] + :or {compressed? true}}] + (thaw ba {:read-eval? read-eval? + :compressor (when compressed? compression/default-snappy-compressor) + :password password + :legacy-mode? true})) \ No newline at end of file diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index d004c36..d56109c 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,6 +1,6 @@ (ns taoensso.nippy.benchmarks {:author "Peter Taoussanis"} - (:require [taoensso.nippy :as nippy :refer (freeze-to-bytes thaw-from-bytes)] + (:require [taoensso.nippy :as nippy :refer (freeze thaw)] [taoensso.nippy.utils :as utils])) ;; Remove stuff from stress-data that breaks reader @@ -8,15 +8,15 @@ (defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 2000)) -(defn reader-freeze [x] (binding [*print-dup* false] (pr-str x))) -(defn reader-thaw [x] (binding [*read-eval* false] (read-string x))) -(def reader-roundtrip (comp reader-thaw reader-freeze)) +(defn freeze-reader [x] (binding [*print-dup* false] (pr-str x))) +(defn thaw-reader [x] (binding [*read-eval* false] (read-string x))) +(def roundtrip-reader (comp freeze-reader thaw-reader)) -(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "p"]) - #(nippy/freeze-to-bytes % :password [:cached "p"]))) -(def roundtrip-fast (comp #(nippy/thaw-from-bytes % :compressed? false) - #(nippy/freeze-to-bytes % :compress? false))) +(def roundtrip-defaults (comp thaw freeze)) +(def roundtrip-encrypted (comp #(thaw % {:password [:cached "p"]}) + #(freeze % {:password [:cached "p"]}))) +(def roundtrip-fast (comp #(thaw % {}) + #(freeze % {:compressor nil}))) (defn autobench [] (println "Benchmarking roundtrips") @@ -35,35 +35,33 @@ (println {:reader - {:freeze (bench (reader-freeze data)) - :thaw (let [frozen (reader-freeze data)] - (bench (reader-thaw frozen))) - :round (bench (reader-roundtrip data)) - :data-size (count (.getBytes ^String (reader-freeze data) "UTF-8"))}}) + {:freeze (bench (freeze-reader data)) + :thaw (let [frozen (freeze-reader data)] (bench (thaw-reader frozen))) + :round (bench (roundtrip-reader data)) + :data-size (count (.getBytes ^String (freeze-reader data) "UTF-8"))}}) (println {:defaults - {:freeze (bench (freeze-to-bytes data)) - :thaw (let [frozen (freeze-to-bytes data)] - (bench (thaw-from-bytes frozen))) - :round (bench (roundtrip-defaults data)) - :data-size (count (freeze-to-bytes data))}}) + {:freeze (bench (freeze data)) + :thaw (let [frozen (freeze data)] (bench (thaw frozen))) + :round (bench (roundtrip-defaults data)) + :data-size (count (freeze data))}}) (println {:encrypted - {:freeze (bench (freeze-to-bytes data :password [:cached "p"])) - :thaw (let [frozen (freeze-to-bytes data :password [:cached "p"])] - (bench (thaw-from-bytes frozen :password [:cached "p"]))) - :round (bench (roundtrip-encrypted data)) - :data-size (count (freeze-to-bytes data :password [:cached "p"]))}}) + {:freeze (bench (freeze data {:password [:cached "p"]})) + :thaw (let [frozen (freeze data {:password [:cached "p"]})] + (bench (thaw frozen {:password [:cached "p"]}))) + :round (bench (roundtrip-encrypted data)) + :data-size (count (freeze data {:password [:cached "p"]}))}}) (println {:fast - {:freeze (bench (freeze-to-bytes data :compress? false)) - :thaw (let [frozen (freeze-to-bytes data :compress? false)] - (bench (thaw-from-bytes frozen :compressed? false))) - :round (bench (roundtrip-fast data)) - :data-size (count (freeze-to-bytes data :compress? false))}}) + {:freeze (bench (freeze data {:compressor nil})) + :thaw (let [frozen (freeze data {:compressor nil})] + (bench (thaw frozen))) + :round (bench (roundtrip-fast data)) + :data-size (count (freeze data {:compressor nil}))}}) (println "Done! (Time for cake?)")) diff --git a/src/taoensso/nippy/encryption.clj b/src/taoensso/nippy/encryption.clj index a0b65ef..8224d45 100644 --- a/src/taoensso/nippy/encryption.clj +++ b/src/taoensso/nippy/encryption.clj @@ -31,7 +31,7 @@ (defn- sha512-key "SHA512-based key generator. Good JVM availability without extra dependencies (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds." - ^javax.crypto.spec.SecretKeySpec [salt-ba ^String pwd] + [salt-ba ^String pwd] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] (if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba)) n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] @@ -78,7 +78,8 @@ key (utils/memoized (when-not salt? (:key-cache this)) sha512-key salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] - (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE key iv) + (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE + ^javax.crypto.spec.SecretKeySpec key iv) (utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba)))) (decrypt [this typed-pwd ba] @@ -91,7 +92,8 @@ key (utils/memoized (when-not salt? (:key-cache this)) sha512-key salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] - (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE key iv) + (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE + ^javax.crypto.spec.SecretKeySpec key iv) (.doFinal aes128-cipher data-ba)))) (def default-aes128-encryptor diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 4fb2b09..a0d4e07 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -1,26 +1,51 @@ (ns taoensso.nippy.tests.main (:require [expectations :as test :refer :all] - [taoensso.nippy :as nippy] + [taoensso.nippy :as nippy :refer (freeze thaw)] [taoensso.nippy.benchmarks :as benchmarks])) ;; Remove stuff from stress-data that breaks roundtrip equality (def test-data (dissoc nippy/stress-data :bytes)) -(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"]) - #(nippy/freeze-to-bytes % :password [:cached "secret"]))) +(def roundtrip-defaults (comp thaw freeze)) +(def roundtrip-encrypted (comp #(thaw % {:password [:salted "p"]}) + #(freeze % {:password [:salted "p"]}))) +(def roundtrip-defaults-legacy (comp #(thaw % {:legacy-mode? true}) + #(freeze % {:legacy-mode? true}))) +(def roundtrip-encrypted-legacy (comp #(thaw % {:password [:salted "p"] + :legacy-mode? true}) + #(freeze % {:password [:salted "p"] + :legacy-mode? true}))) -(expect-focused test-data (roundtrip-defaults test-data)) ; TODO -(expect test-data (roundtrip-encrypted test-data)) -#_(expect ; Snappy lib compatibility ; TODO - (let [thaw #(nippy/thaw-from-bytes % :compressed? false) - ^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false) - ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) - ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] - (= (thaw raw-ba) - (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) - (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) - (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) - (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) +;;; Basic data integrity +(expect test-data (roundtrip-defaults test-data)) +(expect test-data (roundtrip-encrypted test-data)) +(expect test-data (roundtrip-defaults-legacy test-data)) +(expect test-data (roundtrip-encrypted-legacy test-data)) -(expect (benchmarks/autobench)) \ No newline at end of file +(expect ; Snappy lib compatibility (for legacy versions of Nippy) + (let [^bytes raw-ba (freeze test-data {:compressor nil}) + ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) + ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] + (= (thaw raw-ba) + (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) + (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) + (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) + (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) + +;;; API stuff + +;; Strict/auto mode - compression +(expect test-data (thaw (freeze test-data {:compressor nil}))) +(expect Exception (thaw (freeze test-data {:compressor nil}) {:strict? true})) + +;; Strict/auto mode - encryption +(expect test-data (thaw (freeze test-data) {:password [:salted "p"]})) +(expect Exception (thaw (freeze test-data) {:password [:salted "p"] :strict? true})) + +;; Encryption - passwords +(expect Exception (thaw (freeze test-data {:password "malformed"}))) +(expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) +(expect test-data (thaw (freeze test-data {:password [:salted "p"]}) + {:password [:salted "p"]})) + +(expect (benchmarks/autobench)) ; Also tests :cached passwords \ No newline at end of file