luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Encryption: micro optimizations, housekeeping

Browse source
This commit is contained in:
Peter Taoussanis 2016-05-09 13:00:12 +07:00
parent 0df6a7b0f3
commit 4c647465f5
3 changed files with 83 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/taoensso/nippy/compression.clj
View file

@ -27,7 +27,7 @@
Read speed: very high. Read speed: very high.
A good general-purpose compressor." A good general-purpose compressor."
(->SnappyCompressor)) (SnappyCompressor.))
(deftype LZMA2Compressor [compression-level] (deftype LZMA2Compressor [compression-level]
;; Compression level ∈ℕ[0,9] (low->high) with 6 LZMA2 default (we use 0) ;; Compression level ∈ℕ[0,9] (low->high) with 6 LZMA2 default (we use 0)
@ -54,7 +54,8 @@
ba (byte-array len-decomp) ba (byte-array len-decomp)
xzs (org.tukaani.xz.XZInputStream. bais)] xzs (org.tukaani.xz.XZInputStream. bais)]
(.read xzs ba 0 len-decomp) (.read xzs ba 0 len-decomp)
(when (not= -1 (.read xzs)) ; Good practice as extra safety measure (if (== -1 (.read xzs)) ; Good practice as extra safety measure
nil
(throw (ex-info "LZMA2 Decompress failed: corrupt data?" {:ba ba}))) (throw (ex-info "LZMA2 Decompress failed: corrupt data?" {:ba ba})))
ba))) ba)))
@ -66,7 +67,7 @@
A specialized compressor for large, low-write data in space-sensitive A specialized compressor for large, low-write data in space-sensitive
environments." environments."
(->LZMA2Compressor 0)) (LZMA2Compressor. 0))
(deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor (deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor
^net.jpountz.lz4.LZ4Decompressor decompressor] ^net.jpountz.lz4.LZ4Decompressor decompressor]
@ -110,16 +111,16 @@
Thanks to Max Penet (@mpenet) for our first implementation, Thanks to Max Penet (@mpenet) for our first implementation,
Ref. https://github.com/mpenet/nippy-lz4" Ref. https://github.com/mpenet/nippy-lz4"
(->LZ4Compressor (.fastCompressor lz4-factory) (LZ4Compressor. (.fastCompressor lz4-factory)
(.fastDecompressor lz4-factory))) (.fastDecompressor lz4-factory)))
(def lz4hc-compressor (def lz4hc-compressor
"Like `lz4-compressor` but trades some write speed for ratio." "Like `lz4-compressor` but trades some write speed for ratio."
(->LZ4Compressor (.highCompressor lz4-factory) (LZ4Compressor. (.highCompressor lz4-factory)
(.fastDecompressor lz4-factory))) (.fastDecompressor lz4-factory)))
(comment (comment
(def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8")) (def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8"))
(defn bench1 [compressor] (defn bench1 [compressor]
{:time (enc/bench 10000 {:nlaps-warmup 10000} {:time (enc/bench 10000 {:nlaps-warmup 10000}
(->> ba-bench (compress compressor) (decompress compressor))) (->> ba-bench (compress compressor) (decompress compressor)))

130
src/taoensso/nippy/encryption.clj
View file

@ -13,17 +13,9 @@
;;;; Default digests, ciphers, etc. ;;;; Default digests, ciphers, etc.
(def ^:private aes128-cipher* (def ^:private aes128-cipher* (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding")))
(enc/thread-local-proxy (def ^:private sha512-md* (enc/thread-local-proxy (java.security.MessageDigest/getInstance "SHA-512")))
(javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))) (def ^:private prng* (enc/thread-local-proxy (java.security.SecureRandom/getInstance "SHA1PRNG")))
(def ^:private sha512-md*
(enc/thread-local-proxy
(java.security.MessageDigest/getInstance "SHA-512")))
(def ^:private prng*
(enc/thread-local-proxy
(java.security.SecureRandom/getInstance "SHA1PRNG")))
(defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*)) (defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*))
(defn- sha512-md ^java.security.MessageDigest [] (.get ^ThreadLocal sha512-md*)) (defn- sha512-md ^java.security.MessageDigest [] (.get ^ThreadLocal sha512-md*))
@ -39,75 +31,81 @@
(defn- sha512-key (defn- sha512-key
"SHA512-based key generator. Good JVM availability without extra dependencies "SHA512-based key generator. Good JVM availability without extra dependencies
(PBKDF2, bcrypt, scrypt, etc.). Decent security when using many rounds." (PBKDF2, bcrypt, scrypt, etc.). Decent security when using many rounds."
;; [salt-ba ^String pwd & [n]]
[salt-ba ^String pwd] ([salt-ba pwd ] (sha512-key salt-ba pwd (* Short/MAX_VALUE (if salt-ba 5 64))))
(let [md (sha512-md)] ([salt-ba pwd ^long n]
(loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] (let [md (sha512-md)
(if salt-ba (enc/ba-concat salt-ba pwd-ba) pwd-ba)) init-ba (let [pwd-ba (.getBytes ^String pwd "UTF-8")]
;; n (or n (* (int Short/MAX_VALUE) (if salt-ba 5 64))) (if salt-ba (enc/ba-concat salt-ba pwd-ba) pwd-ba))
n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] ^bytes ba (enc/reduce-n (fn [acc in] (.digest md acc)) init-ba n)]
(if-not (zero? n)
(recur (.digest md ba) (dec n)) (-> ba
(-> ba (java.util.Arrays/copyOf aes128-block-size) (java.util.Arrays/copyOf aes128-block-size)
(javax.crypto.spec.SecretKeySpec. "AES")))))) (javax.crypto.spec.SecretKeySpec. "AES")))))
(comment (comment
(time (sha512-key nil "hi" 1)) ; ~40ms per hash (fast) (enc/qb 10
(time (sha512-key nil "hi" 5)) ; ~180ms (default) (sha512-key nil "hi" (* Short/MAX_VALUE 1)) ; ~40ms per hash (fast)
(time (sha512-key nil "hi" 32)) ; ~1200ms (conservative) (sha512-key nil "hi" (* Short/MAX_VALUE 5)) ; ~180ms (default)
(time (sha512-key nil "hi" 128)) ; ~4500ms (paranoid) (sha512-key nil "hi" (* Short/MAX_VALUE 32)) ; ~1200ms (conservative)
) (sha512-key nil "hi" (* Short/MAX_VALUE 128)) ; ~4500ms (paranoid)
))
;;;; Default implementations ;;;; Default implementations
(defn- throw-destructure-ex [typed-password]
(throw (ex-info
(str "Expected password form: "
"[<#{:salted :cached}> <password-string>].\n "
"See `default-aes128-encryptor` docstring for details!")
{:typed-password typed-password})))
(defn- destructure-typed-pwd [typed-password] (defn- destructure-typed-pwd [typed-password]
(let [throw-ex (if (vector? typed-password)
(fn [] (throw (ex-info (let [[type password] typed-password]
(str "Expected password form: " (if (#{:salted :cached} type)
"[<#{:salted :cached}> <password-string>].\n " [type password]
"See `default-aes128-encryptor` docstring for details!") (throw-destructure-ex typed-password)))
{:typed-password typed-password})))] (throw-destructure-ex typed-password)))
(if-not (vector? typed-password) (throw-ex)
(let [[type password] typed-password]
(if-not (#{:salted :cached} type) (throw-ex)
[type password])))))
(comment (destructure-typed-pwd [:salted "foo"])) (comment (destructure-typed-pwd [:salted "foo"]))
(defrecord AES128Encryptor [key-gen key-cache] (defrecord AES128Encryptor [header-id keyfn cached-keyfn]
IEncryptor IEncryptor
(header-id [_] (if (identical? key-gen :sha512) :aes128-sha512 :aes128-other)) (header-id [_] header-id)
(encrypt [_ typed-pwd data-ba] (encrypt [_ typed-pwd data-ba]
(let [[type pwd] (destructure-typed-pwd typed-pwd) (let [[type pwd] (destructure-typed-pwd typed-pwd)
salt? (identical? type :salted) salt? (identical? type :salted)
iv-ba (rand-bytes aes128-block-size) iv-ba (rand-bytes aes128-block-size)
salt-ba (when salt? (rand-bytes salt-size)) salt-ba (when salt? (rand-bytes salt-size))
prefix-ba (if-not salt? iv-ba (enc/ba-concat iv-ba salt-ba)) prefix-ba (if salt? (enc/ba-concat iv-ba salt-ba) iv-ba)
key-gen (if (identical? key-gen :sha512) sha512-key key-gen) key (if salt?
key (if salt? (keyfn salt-ba pwd)
(key-gen salt-ba pwd) (cached-keyfn salt-ba pwd))
(enc/memoized key-cache key-gen salt-ba pwd))
iv (javax.crypto.spec.IvParameterSpec. iv-ba) iv (javax.crypto.spec.IvParameterSpec. iv-ba)
cipher (aes128-cipher)] cipher (aes128-cipher)]
(.init cipher javax.crypto.Cipher/ENCRYPT_MODE (.init cipher javax.crypto.Cipher/ENCRYPT_MODE
^javax.crypto.spec.SecretKeySpec key iv) ^javax.crypto.spec.SecretKeySpec key iv)
(enc/ba-concat prefix-ba (.doFinal cipher data-ba)))) (enc/ba-concat prefix-ba (.doFinal cipher data-ba))))
(decrypt [_ typed-pwd ba] (decrypt [_ typed-pwd ba]
(let [[type pwd] (destructure-typed-pwd typed-pwd) (let [[type pwd] (destructure-typed-pwd typed-pwd)
salt? (= type :salted) salt? (identical? type :salted)
prefix-size (+ aes128-block-size (if salt? salt-size 0)) prefix-size (+ aes128-block-size (if salt? salt-size 0))
[prefix-ba data-ba] (enc/ba-split ba prefix-size) [prefix-ba data-ba] (enc/ba-split ba prefix-size)
[iv-ba salt-ba] (if-not salt? [prefix-ba nil] [iv-ba salt-ba] (if salt?
(enc/ba-split prefix-ba aes128-block-size)) (enc/ba-split prefix-ba aes128-block-size)
key-gen (if (identical? key-gen :sha512) sha512-key key-gen) [prefix-ba nil])
key (if salt? key (if salt?
(key-gen salt-ba pwd) (keyfn salt-ba pwd)
(enc/memoized key-cache key-gen salt-ba pwd)) (cached-keyfn salt-ba pwd))
iv (javax.crypto.spec.IvParameterSpec. iv-ba)
iv (javax.crypto.spec.IvParameterSpec. iv-ba)
cipher (aes128-cipher)] cipher (aes128-cipher)]
(.init cipher javax.crypto.Cipher/DECRYPT_MODE (.init cipher javax.crypto.Cipher/DECRYPT_MODE
^javax.crypto.spec.SecretKeySpec key iv) ^javax.crypto.spec.SecretKeySpec key iv)
(.doFinal cipher data-ba)))) (.doFinal cipher data-ba))))
(def aes128-encryptor (def aes128-encryptor
@ -145,7 +143,8 @@
Faster than `aes128-salted`, and harder to attack any particular key - but Faster than `aes128-salted`, and harder to attack any particular key - but
increased danger if a key is somehow compromised." increased danger if a key is somehow compromised."
(->AES128Encryptor :sha512 (atom {})))
(AES128Encryptor. :aes128-sha512 sha512-key (enc/memoize_ sha512-key)))
;;;; Default implementation ;;;; Default implementation
@ -153,11 +152,14 @@
(def dae aes128-encryptor) (def dae aes128-encryptor)
(def secret-ba (.getBytes "Secret message" "UTF-8")) (def secret-ba (.getBytes "Secret message" "UTF-8"))
(encrypt dae "p" secret-ba) ; Malformed (encrypt dae "p" secret-ba) ; Malformed
(time (encrypt dae [:salted "p"] secret-ba)) (enc/qb 10
(time (encrypt dae [:cached "p"] secret-ba)) (encrypt dae [:salted "p"] secret-ba)
(time (->> secret-ba (encrypt dae [:cached "p"] secret-ba))
(encrypt dae [:salted "p"])
(encrypt dae [:cached "p"]) (enc/qb 10
(decrypt dae [:cached "p"]) (->> secret-ba
(decrypt dae [:salted "p"]) (encrypt dae [:salted "p"])
(String.)))) (encrypt dae [:cached "p"])
(decrypt dae [:cached "p"])
(decrypt dae [:salted "p"])
(String.))))

16
test/taoensso/nippy/tests/main.clj
View file

@ -86,21 +86,21 @@
(is (thrown? Exception ; No thaw extension yet (is (thrown? Exception ; No thaw extension yet
(do (nippy/swap-custom-readers! (constantly {})) (do (nippy/swap-custom-readers! (constantly {}))
(nippy/extend-freeze MyType 1 [x s] (.writeUTF s (.data x))) (nippy/extend-freeze MyType 1 [x s] (.writeUTF s (.data x)))
(thaw (freeze (->MyType "val")))))) (thaw (freeze (MyType. "val"))))))
(is (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s))) (is (do (nippy/extend-thaw 1 [s] (MyType. (.readUTF s)))
(let [mt (->MyType "val")] (= (.data ^MyType mt) (let [mt (MyType. "val")] (= (.data ^MyType mt)
(.data ^MyType (thaw (freeze mt))))))) (.data ^MyType (thaw (freeze mt)))))))
;;; Extend to custom Record ;;; Extend to custom Record
(is (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "foo-" (:data x)))) (is (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "foo-" (:data x))))
(nippy/extend-thaw 2 [s] (->MyRec (.readUTF s))) (nippy/extend-thaw 2 [s] (MyRec. (.readUTF s)))
(= (->MyRec "foo-val") (thaw (freeze (->MyRec "val")))))) (= (MyRec. "foo-val") (thaw (freeze (MyRec. "val"))))))
;;; Keyword (prefixed) extensions ;;; Keyword (prefixed) extensions
(is (is
(do (nippy/extend-freeze MyRec :nippy-tests/MyRec [x s] (.writeUTF s (:data x))) (do (nippy/extend-freeze MyRec :nippy-tests/MyRec [x s] (.writeUTF s (:data x)))
(nippy/extend-thaw :nippy-tests/MyRec [s] (->MyRec (.readUTF s))) (nippy/extend-thaw :nippy-tests/MyRec [s] (MyRec. (.readUTF s)))
(let [mr (->MyRec "val")] (= mr (thaw (freeze mr))))))) (let [mr (MyRec. "val")] (= mr (thaw (freeze mr)))))))
;;;; Caching ;;;; Caching