NB: Simpler, more flexible API (backwards-compatible)
This commit is contained in:
Peter Taoussanis
parent
284d11c660
commit
8d48ec9d75
4 changed files with 201 additions and 89 deletions
|
|
@ -1,16 +1,28 @@
|
|||
(ns taoensso.nippy
|
||||
"Simple, high-performance Clojure serialization library. Adapted from
|
||||
Deep-Freeze."
|
||||
"Simple, high-performance Clojure serialization library. Originally adapted
|
||||
from Deep-Freeze."
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.nippy.utils :as utils]
|
||||
[taoensso.nippy.crypto :as crypto])
|
||||
(:require [taoensso.nippy
|
||||
(utils :as utils)
|
||||
(compression :as compression)
|
||||
(encryption :as encryption)])
|
||||
(:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream
|
||||
ByteArrayInputStream]
|
||||
[clojure.lang Keyword BigInt Ratio PersistentQueue PersistentTreeMap
|
||||
PersistentTreeSet IPersistentList IPersistentVector IPersistentMap
|
||||
IPersistentSet IPersistentCollection]))
|
||||
|
||||
;;;; Header IDs ; TODO
|
||||
;; TODO Allow ba or wrapped-ba input?
|
||||
;; TODO Provide ToFreeze, Frozen, Encrypted, etc. tooling helpers
|
||||
|
||||
;;;; Header IDs
|
||||
;; Nippy 2.x+ prefixes frozen data with a 5-byte header:
|
||||
|
||||
(def ^:const id-nippy-magic-prefix (byte 17))
|
||||
(def ^:const id-nippy-header-ver (byte 0))
|
||||
;; * Compressor id (0 if no compressor)
|
||||
;; * Encryptor id (0 if no encryptor)
|
||||
(def ^:const id-nippy-reserved (byte 0))
|
||||
|
||||
;;;; Data type IDs
|
||||
|
||||
|
|
@ -159,7 +171,29 @@
|
|||
;; Use Clojure's own reader as final fallback
|
||||
(freezer Object id-reader (write-bytes s (.getBytes (pr-str x) "UTF-8")))
|
||||
|
||||
;; TODO New `freeze` API
|
||||
(defn- wrap-nippy-header [data-ba compressor encryptor password]
|
||||
(let [header-ba (byte-array
|
||||
[id-nippy-magic-prefix
|
||||
id-nippy-header-ver
|
||||
(byte (if compressor (compression/header-id compressor) 0))
|
||||
(byte (if password (encryption/header-id encryptor) 0))
|
||||
id-nippy-reserved])]
|
||||
(utils/ba-concat header-ba data-ba)))
|
||||
|
||||
(defn freeze
|
||||
"Serializes arg (any Clojure data type) to a byte array. Enable
|
||||
`:legacy-mode?` flag to produce bytes readable by Nippy < 2.x."
|
||||
^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode?]
|
||||
:or {print-dup? true
|
||||
compressor compression/default-snappy-compressor
|
||||
encryptor encryption/default-aes128-encryptor}}]]
|
||||
(let [ba (ByteArrayOutputStream.)
|
||||
stream (DataOutputStream. ba)]
|
||||
(binding [*print-dup* print-dup?] (freeze-to-stream x stream))
|
||||
(let [ba (.toByteArray ba)
|
||||
ba (if compressor (compression/compress compressor ba) ba)
|
||||
ba (if password (encryption/encrypt encryptor password ba) ba)]
|
||||
(if legacy-mode? ba (wrap-nippy-header ba compressor encryptor password)))))
|
||||
|
||||
;;;; Thawing
|
||||
|
||||
|
|
@ -224,7 +258,80 @@
|
|||
|
||||
(throw (Exception. (str "Failed to thaw unknown type ID: " type-id))))))
|
||||
|
||||
;; TODO New `thaw` API
|
||||
(defn thaw
|
||||
"Deserializes frozen bytes to their original Clojure data type. Enable
|
||||
`:legacy-mode?` to read bytes written by Nippy < 2.x.
|
||||
|
||||
WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless
|
||||
you are sure you know what you're doing."
|
||||
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode?
|
||||
strict?]
|
||||
:or {compressor compression/default-snappy-compressor
|
||||
encryptor encryption/default-aes128-encryptor}}]]
|
||||
|
||||
(let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed. " msg) e)))
|
||||
thaw-data (fn [data-ba compressor password]
|
||||
(let [ba data-ba
|
||||
ba (if password (encryption/decrypt encryptor password ba) ba)
|
||||
ba (if compressor (compression/decompress compressor ba) ba)
|
||||
stream (DataInputStream. (ByteArrayInputStream. ba))]
|
||||
(binding [*read-eval* read-eval?] (thaw-from-stream stream))))]
|
||||
|
||||
(if legacy-mode? ; Nippy < 2.x
|
||||
(try (thaw-data ba compressor password)
|
||||
(catch Exception e
|
||||
(cond password (ex "Unencrypted data or wrong password?" e)
|
||||
compressor (ex "Encrypted or uncompressed data?" e)
|
||||
:else (ex "Encrypted and/or compressed data?" e))))
|
||||
|
||||
;; Nippy >= 2.x, we have a header!
|
||||
(let [[[id-magic* id-header* id-comp* id-enc* _] data-ba]
|
||||
(utils/ba-split ba 5)
|
||||
|
||||
compressed? (not (zero? id-comp*))
|
||||
encrypted? (not (zero? id-enc*))]
|
||||
|
||||
(cond
|
||||
(not= id-magic* id-nippy-magic-prefix)
|
||||
(ex (str "Not Nippy data, data frozen with Nippy < 2.x, "
|
||||
"or data may be corrupt?\n"
|
||||
"Enable `:legacy-mode?` option for data frozen with Nippy < 2.x."))
|
||||
|
||||
(> id-header* id-nippy-header-ver)
|
||||
(ex "Data frozen with newer Nippy version. Please upgrade.")
|
||||
|
||||
(and strict? (not encrypted?) password)
|
||||
(ex (str "Data is not encrypted. Try again w/o password.\n"
|
||||
"Disable `:strict?` option to ignore this error. "))
|
||||
|
||||
(and strict? (not compressed?) compressor)
|
||||
(ex (str "Data is not compressed. Try again w/o compressor.\n"
|
||||
"Disable `:strict?` option to ignore this error."))
|
||||
|
||||
(and encrypted? (not password))
|
||||
(ex "Data is encrypted. Please try again with a password.")
|
||||
|
||||
(and encrypted? password
|
||||
(not= id-enc* (encryption/header-id encryptor)))
|
||||
(ex "Data encrypted with a different Encrypter.")
|
||||
|
||||
(and compressed? compressor
|
||||
(not= id-comp* (compression/header-id compressor)))
|
||||
(ex "Data compressed with a different Compressor.")
|
||||
|
||||
:else
|
||||
(try (thaw-data data-ba (when compressed? compressor)
|
||||
(when encrypted? password))
|
||||
(catch Exception e
|
||||
(if (and encrypted? password)
|
||||
(ex "Wrong password, or data may be corrupt?" e)
|
||||
(ex "Data may be corrupt?" e)))))))))
|
||||
|
||||
(comment (thaw (freeze "hello"))
|
||||
(thaw (freeze "hello" {:compressor nil}))
|
||||
(thaw (freeze "hello" {:compressor nil}) {:strict? true}) ; ex
|
||||
(thaw (freeze "hello" {:password [:salted "p"]})) ; ex
|
||||
(thaw (freeze "hello") {:password [:salted "p"]}))
|
||||
|
||||
;;;; Stress data
|
||||
|
||||
|
|
@ -276,39 +383,19 @@
|
|||
|
||||
;;;; Deprecated API
|
||||
|
||||
;; TODO Rewrite in :legacy terms
|
||||
(defn freeze-to-bytes "DEPRECATED: Use `freeze` instead."
|
||||
^bytes [x & {:keys [compress? print-dup? password]
|
||||
:or {compress? true
|
||||
print-dup? true}}]
|
||||
(let [ba (ByteArrayOutputStream.)
|
||||
stream (DataOutputStream. ba)]
|
||||
(binding [*print-dup* print-dup?] (freeze-to-stream x stream))
|
||||
(let [ba (.toByteArray ba)
|
||||
ba (if compress? (utils/compress-snappy ba) ba)
|
||||
ba (if password (crypto/encrypt-aes128 password ba) ba)]
|
||||
ba)))
|
||||
^bytes [x & {:keys [print-dup? compress? password]
|
||||
:or {print-dup? true
|
||||
compress? true}}]
|
||||
(freeze x {:print-dup? print-dup?
|
||||
:compressor (when compress? compression/default-snappy-compressor)
|
||||
:password password
|
||||
:legacy-mode? true}))
|
||||
|
||||
;; TODO Rewrite in :legacy terms
|
||||
(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead."
|
||||
[ba & {:keys [compressed? read-eval? password]
|
||||
:or {read-eval? false ; For `read-string` injection safety - NB!!!
|
||||
compressed? true}}]
|
||||
(try
|
||||
(let [ba (if password (crypto/decrypt-aes128 password ba) ba)
|
||||
ba (if compressed? (utils/uncompress-snappy ba) ba)
|
||||
stream (DataInputStream. (ByteArrayInputStream. ba))]
|
||||
(binding [*read-eval* read-eval?] (thaw-from-stream stream)))
|
||||
(catch Exception e
|
||||
(throw (Exception.
|
||||
(cond password "Thaw failed. Unencrypted data or bad password?"
|
||||
compressed? "Thaw failed. Encrypted or uncompressed data?"
|
||||
:else "Thaw failed. Encrypted and/or compressed data?")
|
||||
e)))))
|
||||
|
||||
(comment
|
||||
;; Errors
|
||||
(-> (freeze-to-bytes "my data" :password [:salted "password"])
|
||||
(thaw-from-bytes))
|
||||
(-> (freeze-to-bytes "my data" :compress? true)
|
||||
(thaw-from-bytes :compressed? false)))
|
||||
[ba & {:keys [read-eval? compressed? password]
|
||||
:or {compressed? true}}]
|
||||
(thaw ba {:read-eval? read-eval?
|
||||
:compressor (when compressed? compression/default-snappy-compressor)
|
||||
:password password
|
||||
:legacy-mode? true}))
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
(ns taoensso.nippy.benchmarks
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.nippy :as nippy :refer (freeze-to-bytes thaw-from-bytes)]
|
||||
(:require [taoensso.nippy :as nippy :refer (freeze thaw)]
|
||||
[taoensso.nippy.utils :as utils]))
|
||||
|
||||
;; Remove stuff from stress-data that breaks reader
|
||||
|
|
@ -8,15 +8,15 @@
|
|||
|
||||
(defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 2000))
|
||||
|
||||
(defn reader-freeze [x] (binding [*print-dup* false] (pr-str x)))
|
||||
(defn reader-thaw [x] (binding [*read-eval* false] (read-string x)))
|
||||
(def reader-roundtrip (comp reader-thaw reader-freeze))
|
||||
(defn freeze-reader [x] (binding [*print-dup* false] (pr-str x)))
|
||||
(defn thaw-reader [x] (binding [*read-eval* false] (read-string x)))
|
||||
(def roundtrip-reader (comp freeze-reader thaw-reader))
|
||||
|
||||
(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes))
|
||||
(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "p"])
|
||||
#(nippy/freeze-to-bytes % :password [:cached "p"])))
|
||||
(def roundtrip-fast (comp #(nippy/thaw-from-bytes % :compressed? false)
|
||||
#(nippy/freeze-to-bytes % :compress? false)))
|
||||
(def roundtrip-defaults (comp thaw freeze))
|
||||
(def roundtrip-encrypted (comp #(thaw % {:password [:cached "p"]})
|
||||
#(freeze % {:password [:cached "p"]})))
|
||||
(def roundtrip-fast (comp #(thaw % {})
|
||||
#(freeze % {:compressor nil})))
|
||||
|
||||
(defn autobench []
|
||||
(println "Benchmarking roundtrips")
|
||||
|
|
@ -35,35 +35,33 @@
|
|||
|
||||
(println
|
||||
{:reader
|
||||
{:freeze (bench (reader-freeze data))
|
||||
:thaw (let [frozen (reader-freeze data)]
|
||||
(bench (reader-thaw frozen)))
|
||||
:round (bench (reader-roundtrip data))
|
||||
:data-size (count (.getBytes ^String (reader-freeze data) "UTF-8"))}})
|
||||
{:freeze (bench (freeze-reader data))
|
||||
:thaw (let [frozen (freeze-reader data)] (bench (thaw-reader frozen)))
|
||||
:round (bench (roundtrip-reader data))
|
||||
:data-size (count (.getBytes ^String (freeze-reader data) "UTF-8"))}})
|
||||
|
||||
(println
|
||||
{:defaults
|
||||
{:freeze (bench (freeze-to-bytes data))
|
||||
:thaw (let [frozen (freeze-to-bytes data)]
|
||||
(bench (thaw-from-bytes frozen)))
|
||||
:round (bench (roundtrip-defaults data))
|
||||
:data-size (count (freeze-to-bytes data))}})
|
||||
{:freeze (bench (freeze data))
|
||||
:thaw (let [frozen (freeze data)] (bench (thaw frozen)))
|
||||
:round (bench (roundtrip-defaults data))
|
||||
:data-size (count (freeze data))}})
|
||||
|
||||
(println
|
||||
{:encrypted
|
||||
{:freeze (bench (freeze-to-bytes data :password [:cached "p"]))
|
||||
:thaw (let [frozen (freeze-to-bytes data :password [:cached "p"])]
|
||||
(bench (thaw-from-bytes frozen :password [:cached "p"])))
|
||||
:round (bench (roundtrip-encrypted data))
|
||||
:data-size (count (freeze-to-bytes data :password [:cached "p"]))}})
|
||||
{:freeze (bench (freeze data {:password [:cached "p"]}))
|
||||
:thaw (let [frozen (freeze data {:password [:cached "p"]})]
|
||||
(bench (thaw frozen {:password [:cached "p"]})))
|
||||
:round (bench (roundtrip-encrypted data))
|
||||
:data-size (count (freeze data {:password [:cached "p"]}))}})
|
||||
|
||||
(println
|
||||
{:fast
|
||||
{:freeze (bench (freeze-to-bytes data :compress? false))
|
||||
:thaw (let [frozen (freeze-to-bytes data :compress? false)]
|
||||
(bench (thaw-from-bytes frozen :compressed? false)))
|
||||
:round (bench (roundtrip-fast data))
|
||||
:data-size (count (freeze-to-bytes data :compress? false))}})
|
||||
{:freeze (bench (freeze data {:compressor nil}))
|
||||
:thaw (let [frozen (freeze data {:compressor nil})]
|
||||
(bench (thaw frozen)))
|
||||
:round (bench (roundtrip-fast data))
|
||||
:data-size (count (freeze data {:compressor nil}))}})
|
||||
|
||||
(println "Done! (Time for cake?)"))
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@
|
|||
(defn- sha512-key
|
||||
"SHA512-based key generator. Good JVM availability without extra dependencies
|
||||
(PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds."
|
||||
^javax.crypto.spec.SecretKeySpec [salt-ba ^String pwd]
|
||||
[salt-ba ^String pwd]
|
||||
(loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")]
|
||||
(if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba))
|
||||
n (* (int Short/MAX_VALUE) (if salt-ba 5 64))]
|
||||
|
|
@ -78,7 +78,8 @@
|
|||
key (utils/memoized (when-not salt? (:key-cache this))
|
||||
sha512-key salt-ba pwd)
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
|
||||
(.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE key iv)
|
||||
(.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE
|
||||
^javax.crypto.spec.SecretKeySpec key iv)
|
||||
(utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba))))
|
||||
|
||||
(decrypt [this typed-pwd ba]
|
||||
|
|
@ -91,7 +92,8 @@
|
|||
key (utils/memoized (when-not salt? (:key-cache this))
|
||||
sha512-key salt-ba pwd)
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
|
||||
(.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE key iv)
|
||||
(.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE
|
||||
^javax.crypto.spec.SecretKeySpec key iv)
|
||||
(.doFinal aes128-cipher data-ba))))
|
||||
|
||||
(def default-aes128-encryptor
|
||||
|
|
|
|||
|
|
@ -1,26 +1,51 @@
|
|||
(ns taoensso.nippy.tests.main
|
||||
(:require [expectations :as test :refer :all]
|
||||
[taoensso.nippy :as nippy]
|
||||
[taoensso.nippy :as nippy :refer (freeze thaw)]
|
||||
[taoensso.nippy.benchmarks :as benchmarks]))
|
||||
|
||||
;; Remove stuff from stress-data that breaks roundtrip equality
|
||||
(def test-data (dissoc nippy/stress-data :bytes))
|
||||
|
||||
(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes))
|
||||
(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"])
|
||||
#(nippy/freeze-to-bytes % :password [:cached "secret"])))
|
||||
(def roundtrip-defaults (comp thaw freeze))
|
||||
(def roundtrip-encrypted (comp #(thaw % {:password [:salted "p"]})
|
||||
#(freeze % {:password [:salted "p"]})))
|
||||
(def roundtrip-defaults-legacy (comp #(thaw % {:legacy-mode? true})
|
||||
#(freeze % {:legacy-mode? true})))
|
||||
(def roundtrip-encrypted-legacy (comp #(thaw % {:password [:salted "p"]
|
||||
:legacy-mode? true})
|
||||
#(freeze % {:password [:salted "p"]
|
||||
:legacy-mode? true})))
|
||||
|
||||
(expect-focused test-data (roundtrip-defaults test-data)) ; TODO
|
||||
(expect test-data (roundtrip-encrypted test-data))
|
||||
#_(expect ; Snappy lib compatibility ; TODO
|
||||
(let [thaw #(nippy/thaw-from-bytes % :compressed? false)
|
||||
^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false)
|
||||
^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
|
||||
^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)]
|
||||
(= (thaw raw-ba)
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress xerial-ba))
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress iq80-ba))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
|
||||
;;; Basic data integrity
|
||||
(expect test-data (roundtrip-defaults test-data))
|
||||
(expect test-data (roundtrip-encrypted test-data))
|
||||
(expect test-data (roundtrip-defaults-legacy test-data))
|
||||
(expect test-data (roundtrip-encrypted-legacy test-data))
|
||||
|
||||
(expect (benchmarks/autobench))
|
||||
(expect ; Snappy lib compatibility (for legacy versions of Nippy)
|
||||
(let [^bytes raw-ba (freeze test-data {:compressor nil})
|
||||
^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
|
||||
^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)]
|
||||
(= (thaw raw-ba)
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress xerial-ba))
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress iq80-ba))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
|
||||
|
||||
;;; API stuff
|
||||
|
||||
;; Strict/auto mode - compression
|
||||
(expect test-data (thaw (freeze test-data {:compressor nil})))
|
||||
(expect Exception (thaw (freeze test-data {:compressor nil}) {:strict? true}))
|
||||
|
||||
;; Strict/auto mode - encryption
|
||||
(expect test-data (thaw (freeze test-data) {:password [:salted "p"]}))
|
||||
(expect Exception (thaw (freeze test-data) {:password [:salted "p"] :strict? true}))
|
||||
|
||||
;; Encryption - passwords
|
||||
(expect Exception (thaw (freeze test-data {:password "malformed"})))
|
||||
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
|
||||
(expect test-data (thaw (freeze test-data {:password [:salted "p"]})
|
||||
{:password [:salted "p"]}))
|
||||
|
||||
(expect (benchmarks/autobench)) ; Also tests :cached passwords
|
||||
Loading…
Reference in a new issue