luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NB: Simpler, more flexible API (backwards-compatible)

Browse source
This commit is contained in:
Peter Taoussanis 2013-06-13 12:12:28 +07:00
parent 284d11c660
commit 8d48ec9d75
4 changed files with 201 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

167
src/taoensso/nippy.clj
View file

@ -1,16 +1,28 @@
(ns taoensso.nippy (ns taoensso.nippy
"Simple, high-performance Clojure serialization library. Adapted from "Simple, high-performance Clojure serialization library. Originally adapted
Deep-Freeze." from Deep-Freeze."
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [taoensso.nippy.utils :as utils] (:require [taoensso.nippy
[taoensso.nippy.crypto :as crypto]) (utils :as utils)
(compression :as compression)
(encryption :as encryption)])
(:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream (:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream
ByteArrayInputStream] ByteArrayInputStream]
[clojure.lang Keyword BigInt Ratio PersistentQueue PersistentTreeMap [clojure.lang Keyword BigInt Ratio PersistentQueue PersistentTreeMap
PersistentTreeSet IPersistentList IPersistentVector IPersistentMap PersistentTreeSet IPersistentList IPersistentVector IPersistentMap
IPersistentSet IPersistentCollection])) IPersistentSet IPersistentCollection]))
;;;; Header IDs ; TODO ;; TODO Allow ba or wrapped-ba input?
;; TODO Provide ToFreeze, Frozen, Encrypted, etc. tooling helpers
;;;; Header IDs
;; Nippy 2.x+ prefixes frozen data with a 5-byte header:
(def ^:const id-nippy-magic-prefix (byte 17))
(def ^:const id-nippy-header-ver (byte 0))
;; * Compressor id (0 if no compressor)
;; * Encryptor id (0 if no encryptor)
(def ^:const id-nippy-reserved (byte 0))
;;;; Data type IDs ;;;; Data type IDs
@ -159,7 +171,29 @@
;; Use Clojure's own reader as final fallback ;; Use Clojure's own reader as final fallback
(freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8"))) (freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8")))
;; TODO New `freeze` API (defn- wrap-nippy-header [data-ba compressor encryptor password]
(let [header-ba (byte-array
[id-nippy-magic-prefix
id-nippy-header-ver
(byte (if compressor (compression/header-id compressor) 0))
(byte (if password (encryption/header-id encryptor) 0))
id-nippy-reserved])]
(utils/ba-concat header-ba data-ba)))
(defn freeze
"Serializes arg (any Clojure data type) to a byte array. Enable
`:legacy-mode?` flag to produce bytes readable by Nippy < 2.x."
^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode?]
:or {print-dup? true
compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]]
(let [ba (ByteArrayOutputStream.)
stream (DataOutputStream. ba)]
(binding [*print-dup* print-dup?] (freeze-to-stream x stream))
(let [ba (.toByteArray ba)
ba (if compressor (compression/compress compressor ba) ba)
ba (if password (encryption/encrypt encryptor password ba) ba)]
(if legacy-mode? ba (wrap-nippy-header ba compressor encryptor password)))))
;;;; Thawing ;;;; Thawing
@ -224,7 +258,80 @@
(throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) (throw (Exception. (str "Failed to thaw unknown type ID: " type-id))))))
;; TODO New `thaw` API (defn thaw
"Deserializes frozen bytes to their original Clojure data type. Enable
`:legacy-mode?` to read bytes written by Nippy < 2.x.
WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless
you are sure you know what you're doing."
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode?
strict?]
:or {compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]]
(let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed. " msg) e)))
thaw-data (fn [data-ba compressor password]
(let [ba data-ba
ba (if password (encryption/decrypt encryptor password ba) ba)
ba (if compressor (compression/decompress compressor ba) ba)
stream (DataInputStream. (ByteArrayInputStream. ba))]
(binding [*read-eval* read-eval?] (thaw-from-stream stream))))]
(if legacy-mode? ; Nippy < 2.x
(try (thaw-data ba compressor password)
(catch Exception e
(cond password (ex "Unencrypted data or wrong password?" e)
compressor (ex "Encrypted or uncompressed data?" e)
:else (ex "Encrypted and/or compressed data?" e))))
;; Nippy >= 2.x, we have a header!
(let [[[id-magic* id-header* id-comp* id-enc* _] data-ba]
(utils/ba-split ba 5)
compressed? (not (zero? id-comp*))
encrypted? (not (zero? id-enc*))]
(cond
(not= id-magic* id-nippy-magic-prefix)
(ex (str "Not Nippy data, data frozen with Nippy < 2.x, "
"or data may be corrupt?\n"
"Enable `:legacy-mode?` option for data frozen with Nippy < 2.x."))
(> id-header* id-nippy-header-ver)
(ex "Data frozen with newer Nippy version. Please upgrade.")
(and strict? (not encrypted?) password)
(ex (str "Data is not encrypted. Try again w/o password.\n"
"Disable `:strict?` option to ignore this error. "))
(and strict? (not compressed?) compressor)
(ex (str "Data is not compressed. Try again w/o compressor.\n"
"Disable `:strict?` option to ignore this error."))
(and encrypted? (not password))
(ex "Data is encrypted. Please try again with a password.")
(and encrypted? password
(not= id-enc* (encryption/header-id encryptor)))
(ex "Data encrypted with a different Encrypter.")
(and compressed? compressor
(not= id-comp* (compression/header-id compressor)))
(ex "Data compressed with a different Compressor.")
:else
(try (thaw-data data-ba (when compressed? compressor)
(when encrypted? password))
(catch Exception e
(if (and encrypted? password)
(ex "Wrong password, or data may be corrupt?" e)
(ex "Data may be corrupt?" e)))))))))
(comment (thaw (freeze "hello"))
(thaw (freeze "hello" {:compressor nil}))
(thaw (freeze "hello" {:compressor nil}) {:strict? true}) ; ex
(thaw (freeze "hello" {:password [:salted "p"]})) ; ex
(thaw (freeze "hello") {:password [:salted "p"]}))
;;;; Stress data ;;;; Stress data
@ -276,39 +383,19 @@
;;;; Deprecated API ;;;; Deprecated API
;; TODO Rewrite in :legacy terms
(defn freeze-to-bytes "DEPRECATED: Use `freeze` instead." (defn freeze-to-bytes "DEPRECATED: Use `freeze` instead."
^bytes [x & {:keys [compress? print-dup? password] ^bytes [x & {:keys [print-dup? compress? password]
:or {compress? true :or {print-dup? true
print-dup? true}}] compress? true}}]
(let [ba (ByteArrayOutputStream.) (freeze x {:print-dup? print-dup?
stream (DataOutputStream. ba)] :compressor (when compress? compression/default-snappy-compressor)
(binding [*print-dup* print-dup?] (freeze-to-stream x stream)) :password password
(let [ba (.toByteArray ba) :legacy-mode? true}))
ba (if compress? (utils/compress-snappy ba) ba)
ba (if password (crypto/encrypt-aes128 password ba) ba)]
ba)))
;; TODO Rewrite in :legacy terms
(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead." (defn thaw-from-bytes "DEPRECATED: Use `thaw` instead."
[ba & {:keys [compressed? read-eval? password] [ba & {:keys [read-eval? compressed? password]
:or {read-eval? false ; For `read-string` injection safety - NB!!! :or {compressed? true}}]
compressed? true}}] (thaw ba {:read-eval? read-eval?
(try :compressor (when compressed? compression/default-snappy-compressor)
(let [ba (if password (crypto/decrypt-aes128 password ba) ba) :password password
ba (if compressed? (utils/uncompress-snappy ba) ba) :legacy-mode? true}))
stream (DataInputStream. (ByteArrayInputStream. ba))]
(binding [*read-eval* read-eval?] (thaw-from-stream stream)))
(catch Exception e
(throw (Exception.
(cond password "Thaw failed. Unencrypted data or bad password?"
compressed? "Thaw failed. Encrypted or uncompressed data?"
:else "Thaw failed. Encrypted and/or compressed data?")
e)))))
(comment
;; Errors
(-> (freeze-to-bytes "my data" :password [:salted "password"])
(thaw-from-bytes))
(-> (freeze-to-bytes "my data" :compress? true)
(thaw-from-bytes :compressed? false)))

50
src/taoensso/nippy/benchmarks.clj
View file

@ -1,6 +1,6 @@
(ns taoensso.nippy.benchmarks (ns taoensso.nippy.benchmarks
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [taoensso.nippy :as nippy :refer (freeze-to-bytes thaw-from-bytes)] (:require [taoensso.nippy :as nippy :refer (freeze thaw)]
[taoensso.nippy.utils :as utils])) [taoensso.nippy.utils :as utils]))
;; Remove stuff from stress-data that breaks reader ;; Remove stuff from stress-data that breaks reader
@ -8,15 +8,15 @@
(defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 2000)) (defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 2000))
(defn reader-freeze [x] (binding [*print-dup* false] (pr-str x))) (defn freeze-reader [x] (binding [*print-dup* false] (pr-str x)))
(defn reader-thaw [x] (binding [*read-eval* false] (read-string x))) (defn thaw-reader [x] (binding [*read-eval* false] (read-string x)))
(def reader-roundtrip (comp reader-thaw reader-freeze)) (def roundtrip-reader (comp freeze-reader thaw-reader))
(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) (def roundtrip-defaults (comp thaw freeze))
(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "p"]) (def roundtrip-encrypted (comp #(thaw % {:password [:cached "p"]})
#(nippy/freeze-to-bytes % :password [:cached "p"]))) #(freeze % {:password [:cached "p"]})))
(def roundtrip-fast (comp #(nippy/thaw-from-bytes % :compressed? false) (def roundtrip-fast (comp #(thaw % {})
#(nippy/freeze-to-bytes % :compress? false))) #(freeze % {:compressor nil})))
(defn autobench [] (defn autobench []
(println "Benchmarking roundtrips") (println "Benchmarking roundtrips")
@ -35,35 +35,33 @@
(println (println
{:reader {:reader
{:freeze (bench (reader-freeze data)) {:freeze (bench (freeze-reader data))
:thaw (let [frozen (reader-freeze data)] :thaw (let [frozen (freeze-reader data)] (bench (thaw-reader frozen)))
(bench (reader-thaw frozen))) :round (bench (roundtrip-reader data))
:round (bench (reader-roundtrip data)) :data-size (count (.getBytes ^String (freeze-reader data) "UTF-8"))}})
:data-size (count (.getBytes ^String (reader-freeze data) "UTF-8"))}})
(println (println
{:defaults {:defaults
{:freeze (bench (freeze-to-bytes data)) {:freeze (bench (freeze data))
:thaw (let [frozen (freeze-to-bytes data)] :thaw (let [frozen (freeze data)] (bench (thaw frozen)))
(bench (thaw-from-bytes frozen)))
:round (bench (roundtrip-defaults data)) :round (bench (roundtrip-defaults data))
:data-size (count (freeze-to-bytes data))}}) :data-size (count (freeze data))}})
(println (println
{:encrypted {:encrypted
{:freeze (bench (freeze-to-bytes data :password [:cached "p"])) {:freeze (bench (freeze data {:password [:cached "p"]}))
:thaw (let [frozen (freeze-to-bytes data :password [:cached "p"])] :thaw (let [frozen (freeze data {:password [:cached "p"]})]
(bench (thaw-from-bytes frozen :password [:cached "p"]))) (bench (thaw frozen {:password [:cached "p"]})))
:round (bench (roundtrip-encrypted data)) :round (bench (roundtrip-encrypted data))
:data-size (count (freeze-to-bytes data :password [:cached "p"]))}}) :data-size (count (freeze data {:password [:cached "p"]}))}})
(println (println
{:fast {:fast
{:freeze (bench (freeze-to-bytes data :compress? false)) {:freeze (bench (freeze data {:compressor nil}))
:thaw (let [frozen (freeze-to-bytes data :compress? false)] :thaw (let [frozen (freeze data {:compressor nil})]
(bench (thaw-from-bytes frozen :compressed? false))) (bench (thaw frozen)))
:round (bench (roundtrip-fast data)) :round (bench (roundtrip-fast data))
:data-size (count (freeze-to-bytes data :compress? false))}}) :data-size (count (freeze data {:compressor nil}))}})
(println "Done! (Time for cake?)")) (println "Done! (Time for cake?)"))

8
src/taoensso/nippy/encryption.clj
View file

@ -31,7 +31,7 @@
(defn- sha512-key (defn- sha512-key
"SHA512-based key generator. Good JVM availability without extra dependencies "SHA512-based key generator. Good JVM availability without extra dependencies
(PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds." (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds."
^javax.crypto.spec.SecretKeySpec [salt-ba ^String pwd] [salt-ba ^String pwd]
(loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")]
(if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba)) (if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba))
n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] n (* (int Short/MAX_VALUE) (if salt-ba 5 64))]
@ -78,7 +78,8 @@
key (utils/memoized (when-not salt? (:key-cache this)) key (utils/memoized (when-not salt? (:key-cache this))
sha512-key salt-ba pwd) sha512-key salt-ba pwd)
iv (javax.crypto.spec.IvParameterSpec. iv-ba)] iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
(.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE key iv) (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE
^javax.crypto.spec.SecretKeySpec key iv)
(utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba)))) (utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba))))
(decrypt [this typed-pwd ba] (decrypt [this typed-pwd ba]
@ -91,7 +92,8 @@
key (utils/memoized (when-not salt? (:key-cache this)) key (utils/memoized (when-not salt? (:key-cache this))
sha512-key salt-ba pwd) sha512-key salt-ba pwd)
iv (javax.crypto.spec.IvParameterSpec. iv-ba)] iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
(.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE key iv) (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE
^javax.crypto.spec.SecretKeySpec key iv)
(.doFinal aes128-cipher data-ba)))) (.doFinal aes128-cipher data-ba))))
(def default-aes128-encryptor (def default-aes128-encryptor

43
test/taoensso/nippy/tests/main.clj
View file

@ -1,20 +1,29 @@
(ns taoensso.nippy.tests.main (ns taoensso.nippy.tests.main
(:require [expectations :as test :refer :all] (:require [expectations :as test :refer :all]
[taoensso.nippy :as nippy] [taoensso.nippy :as nippy :refer (freeze thaw)]
[taoensso.nippy.benchmarks :as benchmarks])) [taoensso.nippy.benchmarks :as benchmarks]))
;; Remove stuff from stress-data that breaks roundtrip equality ;; Remove stuff from stress-data that breaks roundtrip equality
(def test-data (dissoc nippy/stress-data :bytes)) (def test-data (dissoc nippy/stress-data :bytes))
(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) (def roundtrip-defaults (comp thaw freeze))
(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"]) (def roundtrip-encrypted (comp #(thaw % {:password [:salted "p"]})
#(nippy/freeze-to-bytes % :password [:cached "secret"]))) #(freeze % {:password [:salted "p"]})))
(def roundtrip-defaults-legacy (comp #(thaw % {:legacy-mode? true})
#(freeze % {:legacy-mode? true})))
(def roundtrip-encrypted-legacy (comp #(thaw % {:password [:salted "p"]
:legacy-mode? true})
#(freeze % {:password [:salted "p"]
:legacy-mode? true})))
(expect-focused test-data (roundtrip-defaults test-data)) ; TODO ;;; Basic data integrity
(expect test-data (roundtrip-defaults test-data))
(expect test-data (roundtrip-encrypted test-data)) (expect test-data (roundtrip-encrypted test-data))
#_(expect ; Snappy lib compatibility ; TODO (expect test-data (roundtrip-defaults-legacy test-data))
(let [thaw #(nippy/thaw-from-bytes % :compressed? false) (expect test-data (roundtrip-encrypted-legacy test-data))
^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false)
(expect ; Snappy lib compatibility (for legacy versions of Nippy)
(let [^bytes raw-ba (freeze test-data {:compressor nil})
^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)]
(= (thaw raw-ba) (= (thaw raw-ba)
@ -23,4 +32,20 @@
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
(expect (benchmarks/autobench)) ;;; API stuff
;; Strict/auto mode - compression
(expect test-data (thaw (freeze test-data {:compressor nil})))
(expect Exception (thaw (freeze test-data {:compressor nil}) {:strict? true}))
;; Strict/auto mode - encryption
(expect test-data (thaw (freeze test-data) {:password [:salted "p"]}))
(expect Exception (thaw (freeze test-data) {:password [:salted "p"] :strict? true}))
;; Encryption - passwords
(expect Exception (thaw (freeze test-data {:password "malformed"})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
(expect test-data (thaw (freeze test-data {:password [:salted "p"]})
{:password [:salted "p"]}))
(expect (benchmarks/autobench)) ; Also tests :cached passwords