luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Crypto: simplify design, add auto salting

Browse source 
Have decided to simplify the API even further and bring configuration down to
essentially one decision: do you want auto salting, or key caching?
This commit is contained in:
Peter Taoussanis 2013-06-11 23:03:30 +07:00
parent bea3f5e84e
commit 4ac2a34d7a
7 changed files with 143 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version:
```clojure
[com.taoensso/nippy "1.2.1"] ; Stable
[com.taoensso/nippy "1.3.0-alpha1"] ; Development (adds crypto support)
[com.taoensso/nippy "1.3.0-alpha2"] ; Development (adds crypto support)
```
# Nippy, a Clojure serialization library
@ -18,7 +18,7 @@ Nippy is an attempt to provide a drop-in, high-performance alternative to the re
* **Reader-fallback** for difficult/future types (including Clojure 1.4+ tagged literals).
* **Full test coverage** for every supported type.
* [Snappy](http://code.google.com/p/snappy/) **integrated de/compression** for efficient storage and network transfer.
* Enable **high-strength encryption** with a single `:password "my-password"` option. (1.3.0+)
* Enable **high-strength encryption** with a single `:password [:salted "my-password"]` option. (1.3.0+)
## Getting started

2
project.clj
View file

@ -1,4 +1,4 @@
(defproject com.taoensso/nippy "1.3.0-alpha1"
(defproject com.taoensso/nippy "1.3.0-alpha2"
:description "Clojure serialization library"
:url "https://github.com/ptaoussanis/nippy"
:license {:name "Eclipse Public License"

15
src/taoensso/nippy.clj
View file

@ -167,16 +167,15 @@
(defn freeze-to-bytes
"Serializes x to a byte array and returns the array."
^bytes [x & {:keys [crypto compress? print-dup? salt password]
:or {crypto crypto/crypto-default
compress? true
^bytes [x & {:keys [compress? print-dup? password]
:or {compress? true
print-dup? true}}]
(let [ba (ByteArrayOutputStream.)
stream (DataOutputStream. ba)]
(freeze-to-stream! stream x print-dup?)
(let [ba (.toByteArray ba)
ba (if compress? (utils/compress-bytes ba) ba)
ba (if password (crypto/encrypt crypto salt password ba) ba)]
ba (if password (crypto/encrypt-aes128 password ba) ba)]
ba)))
;;;; Thawing
@ -255,11 +254,11 @@
(defn thaw-from-bytes
"Deserializes an object from given byte array."
[ba & {:keys [crypto compressed? read-eval? salt password]
:or {crypto crypto/crypto-default
[ba & {:keys [compressed? read-eval? password]
:or {compressed? true
read-eval? false ; For `read-string` injection safety - NB!!!
compressed? true}}]
(-> (let [ba (if password (crypto/decrypt crypto salt password ba) ba)
}}]
(-> (let [ba (if password (crypto/decrypt-aes128 password ba) ba)
ba (if compressed? (utils/uncompress-bytes ba) ba)]
ba)
(ByteArrayInputStream.)

16
src/taoensso/nippy/benchmarks.clj
View file

@ -13,11 +13,9 @@
(defn reader-thaw [x] (binding [*read-eval* false] (read-string x)))
(def reader-roundtrip (comp reader-thaw reader-freeze))
(def crypto-opts [:password "secret" :crypto crypto/crypto-default-cached])
(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes))
(def roundtrip-encrypted (comp #(apply nippy/thaw-from-bytes % crypto-opts)
#(apply nippy/freeze-to-bytes % crypto-opts)))
(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "p"])
#(nippy/freeze-to-bytes % :password [:cached "p"])))
(def roundtrip-fast (comp #(nippy/thaw-from-bytes % :compressed? false)
#(nippy/freeze-to-bytes % :compress? false)))
@ -48,11 +46,11 @@
(println
{:encrypted
{:freeze (bench (apply freeze-to-bytes data crypto-opts))
:thaw (let [frozen (apply freeze-to-bytes data crypto-opts)]
(bench (apply thaw-from-bytes frozen crypto-opts)))
{:freeze (bench (freeze-to-bytes data :password [:cached "p"]))
:thaw (let [frozen (freeze-to-bytes data :password [:cached "p"])]
(bench (thaw-from-bytes frozen :password [:cached "p"])))
:round (bench (roundtrip-encrypted data))
:data-size (count (apply freeze-to-bytes data crypto-opts))}})
:data-size (count (freeze-to-bytes data :password [:cached "p"]))}})
(println
{:fast
@ -66,9 +64,9 @@
;;; 11 June 2013: Clojure 1.5.1, Nippy 1.3.0-alpha1
;; {:reader {:freeze 17042, :thaw 31579, :round 48379, :data-size 22954}}
;; {:fast {:freeze 3078, :thaw 4684, :round 8117, :data-size 13274}}
;; {:defaults {:freeze 3810, :thaw 5295, :round 9052, :data-size 12394}}
;; {:encrypted {:freeze 5800, :thaw 6862, :round 12317, :data-size 12416}}
;; {:fast {:freeze 3078, :thaw 4684, :round 8117, :data-size 13274}}
;;; Clojure 1.5.1, Nippy 1.2.1 (+ sorted-set, sorted-map)
;; (def data (dissoc data :sorted-set :sorted-map))

200
src/taoensso/nippy/crypto.clj
View file

@ -3,113 +3,145 @@
Simple no-nonsense crypto with reasonable defaults. Because your Clojure data
deserves some privacy."
{:author "Peter Taoussanis"}
(:require [taoensso.nippy.utils :as utils]))
(:require [clojure.string :as str]
[taoensso.nippy.utils :as utils]))
(defprotocol ICrypto "Simple cryptography interface."
(gen-key ^javax.crypto.spec.SecretKeySpec [crypto salt pwd]
"Returns an appropriate SecretKeySpec.")
(encrypt ^bytes [crypto salt pwd ba] "Returns encrypted bytes.")
(decrypt ^bytes [crypto salt pwd ba] "Returns decrypted bytes."))
;;;; Interface
(defrecord CryptoAES [cipher-type default-salt key-gen-opts cache])
(defprotocol IEncrypter
(gen-key ^javax.crypto.spec.SecretKeySpec [encrypter salt-ba pwd])
(encrypt ^bytes [encrypter pwd ba])
(decrypt ^bytes [encrypter pwd ba]))
(def ^:private ^java.security.MessageDigest sha-md
(java.security.MessageDigest/getInstance "SHA-512"))
(defrecord AES128Encrypter [key-work-factor key-cache])
(def ^:private ^:const aes128-block-size (int 16))
;;;; Digests, ciphers, etc.
(defn- sha512-key
"Default SHA512-based key generator. Good JVM availability without extra
dependencies (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple
rounds. VERY aggressive multiples (>64) possible+recommended when cached."
[^String salted-pwd & [{:keys [rounds-multiple]
:or {rounds-multiple 5}}]] ; Cacheable
(loop [^bytes ba (.getBytes salted-pwd "UTF-8")
n (* (int Short/MAX_VALUE) (or rounds-multiple 5))]
(if-not (zero? n)
(recur (.digest sha-md ba) (dec n))
(-> ba
;; 128bit keys have good JVM availability and are
;; entirely sufficient, Ref. http://goo.gl/2YRQG
(java.util.Arrays/copyOf aes128-block-size)
(def ^:private ^:const aes128-block-size (int 16))
(def ^:private ^:const salt-size (int 16))
(def ^:private ^javax.crypto.Cipher aes128-cipher
(javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))
(def ^:private ^java.security.MessageDigest sha512-md
(java.security.MessageDigest/getInstance "SHA-512"))
(def ^:private ^java.security.SecureRandom prng
(java.security.SecureRandom/getInstance "SHA1PRNG"))
(defn- rand-bytes [size] (let [seed (byte-array size)] (.nextBytes prng seed) seed))
;;;; Default keygen
(defn- sha512-key
"SHA512-based key generator. Good JVM availability without extra dependencies
(PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds."
[salt-ba ^String pwd key-work-factor]
(loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")]
(if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba))
n (* (int Short/MAX_VALUE) key-work-factor)]
(if-not (zero? n)
(recur (.digest sha512-md ba) (dec n))
(-> ba (java.util.Arrays/copyOf aes128-block-size)
(javax.crypto.spec.SecretKeySpec. "AES")))))
(comment
(time (sha512-key "hi" {:rounds-multiple 1})) ; ~40ms per hash (fast)
(time (sha512-key "hi" {:rounds-multiple 5})) ; ~180ms (default)
(time (sha512-key "hi" {:rounds-multiple 32})) ; ~1200ms (conservative)
(time (sha512-key "hi" {:rounds-multiple 128})) ; ~4500ms (paranoid)
(time (sha512-key nil "hi" 1)) ; ~40ms per hash (fast)
(time (sha512-key nil "hi" 5)) ; ~180ms (default)
(time (sha512-key nil "hi" 32)) ; ~1200ms (conservative)
(time (sha512-key nil "hi" 128)) ; ~4500ms (paranoid)
)
(def ^:private cipher* (memoize #(javax.crypto.Cipher/getInstance %)))
(defn- cipher ^javax.crypto.Cipher [cipher-type] (cipher* cipher-type))
;;;; Default implementation
(def ^:private ^java.security.SecureRandom rand-gen
(java.security.SecureRandom/getInstance "SHA1PRNG"))
(defn- rand-bytes [size] (let [seed (make-array Byte/TYPE size)]
(.nextBytes rand-gen seed) seed))
(extend-type AES128Encrypter
IEncrypter
(gen-key [{:keys [key-work-factor key-cache]} salt-ba pwd]
;; Trade-off: salt-ba and key-cache mutually exclusive
(utils/memoized key-cache sha512-key salt-ba pwd key-work-factor))
(extend-type CryptoAES
ICrypto
(gen-key [{:keys [default-salt key-gen-opts cache]} salt pwd]
(utils/apply-memoized cache
sha512-key (str (or salt default-salt) pwd) key-gen-opts))
(encrypt [{:keys [cipher-type cache] :as crypto} salt pwd ba]
(let [cipher (cipher cipher-type)
key (gen-key crypto salt pwd)
(encrypt [{:keys [key-cache] :as this} pwd data-ba]
(let [salt? (not key-cache)
iv-ba (rand-bytes aes128-block-size)
salt-ba (when salt? (rand-bytes salt-size))
prefix-ba (if-not salt? iv-ba (utils/ba-concat iv-ba salt-ba))
key (gen-key this salt-ba pwd)
iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
(.init cipher javax.crypto.Cipher/ENCRYPT_MODE key iv)
(utils/ba-concat iv-ba (.doFinal cipher ba))))
(.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE key iv)
(utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba))))
(decrypt [{:keys [cipher-type cache] :as crypto} salt pwd ba]
(let [cipher (cipher cipher-type)
key (gen-key crypto salt pwd)
[iv-ba data-ba] (utils/ba-split ba aes128-block-size)
(decrypt [{:keys [key-cache] :as this} pwd ba]
(let [salt? (not key-cache)
prefix-size (+ aes128-block-size (if salt? salt-size 0))
[prefix-ba data-ba] (utils/ba-split ba prefix-size)
[iv-ba salt-ba] (if-not salt? [prefix-ba nil]
(utils/ba-split prefix-ba aes128-block-size))
key (gen-key this salt-ba pwd)
iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
(.init cipher javax.crypto.Cipher/DECRYPT_MODE key iv)
(.doFinal cipher data-ba))))
(.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE key iv)
(.doFinal aes128-cipher data-ba))))
(defn crypto-aes128
"Returns a new CryptoAES object with options:
:default-salt - Shared fallback password salt when none is provided. If
the use case allows it, a unique random salt per
encrypted item is better.
:cache-keys? - IMPORTANT. DO enable this if and ONLY if your use case
involves only a small, finite number of unique secret
keys (salt+password)s. Dramatically improves `gen-key`
performance in those cases and (as a result) allows for
a *much* stronger `key-work-factor`.
:key-work-factor - O(n) CPU time needed to generate keys. Larger factors
provide more protection against brute-force attacks but
make encryption+decryption slower if `:cache-keys?` is
not enabled.
(def aes128-salted
"USE CASE: You want more than a small, finite number of passwords (e.g. each
item encrypted will use a unique user-provided password).
Some sensible values (from fast to strong):
Without caching: 1, 5, 10
With caching: 5, 32, 64, 128
IMPLEMENTATION: Uses a relatively cheap key hash, but automatically salts
every key.
See also `crypto-default` and `crypto-default-cached` for sensible ready-made
CryptoAES objects."
[& [{:keys [default-salt cache-keys? key-work-factor]
:or {default-salt "XA~I3(:]3'ck5!M[z\\m`l^0mltR~y/]Arq_d9+$`e#yJssN^8"
key-work-factor 5}}]]
(CryptoAES. "AES/CBC/PKCS5Padding"
default-salt
{:rounds-multiple (int key-work-factor)}
(when cache-keys? (atom {}))))
PROS: Each key is independent so would need to be attacked independently.
CONS: Key caching impossible, so there's an inherent trade-off between
encryption/decryption speed and the difficulty of attacking any
particular key.
(def crypto-default (crypto-aes128))
(def crypto-default-cached (crypto-aes128 {:cache-keys? true
:key-work-factor 64}))
Slower than `aes128-cached`, and easier to attack any particular key."
(AES128Encrypter. 5 nil))
(def aes128-cached
"USE CASE: You want only a small, finite number of passwords (e.g. a limited
number of staff/admins, or you'll be using a single password to
encrypt many items).
IMPLEMENTATION: Uses a _very_ expensive (but cached) key hash, and no salt.
PROS: Great amortized encryption/decryption speed. Expensive key hash makes
attacking any particular key very difficult.
CONS: Using a small number of keys for many encrypted items means that if any
key _is_ somehow compromised, _all_ items encrypted with that key are
compromised.
Faster than `aes128-salted`, and harder to attack any particular key - but
increased danger if a key is somehow compromised."
(AES128Encrypter. 64 (atom {})))
(defn- destructure-typed-password
"[<type> <password>] -> [Encrypter <password>]"
[typed-password]
(letfn [(throw-ex []
(throw (Exception.
(str "Expected password form: "
"[<#{:salted :cached}> <password-string>].\n "
"See `aes128-salted`, `aes128-cached` for details."))))]
(if-not (vector? typed-password)
(throw-ex)
(let [[type password] typed-password]
[(case type :salted aes128-salted :cached aes128-cached (throw-ex))
password]))))
(defn encrypt-aes128 [typed-password ba]
(let [[encrypter password] (destructure-typed-password typed-password)]
(encrypt encrypter password ba)))
(defn decrypt-aes128 [typed-password ba]
(let [[encrypter password] (destructure-typed-password typed-password)]
(decrypt encrypter password ba)))
(comment
(time (gen-key crypto-default "my-salt" "my-password"))
(time (gen-key crypto-default-cached "my-salt" "my-password"))
(encrypt-aes128 "my-password" (.getBytes "Secret message")) ; Malformed
(time (gen-key aes128-salted nil "my-password"))
(time (gen-key aes128-cached nil "my-password"))
(time (->> (.getBytes "Secret message" "UTF-8")
(encrypt crypto-default "s" "p")
(encrypt crypto-default "s" "p")
(decrypt crypto-default "s" "p")
(decrypt crypto-default "s" "p")
(encrypt-aes128 [:salted "p"])
(encrypt-aes128 [:cached "p"])
(decrypt-aes128 [:cached "p"])
(decrypt-aes128 [:salted "p"])
(String.))))

9
src/taoensso/nippy/utils.clj
View file

@ -61,9 +61,9 @@
(defn compress-bytes [^bytes ba] (Snappy/compress ba))
(defn uncompress-bytes [^bytes ba] (Snappy/uncompress ba 0 (alength ba)))
(defn apply-memoized
"A cross between `memoize` and `apply`. Operates like `apply` but accepts an
optional {<args> <value> ...} cache atom."
(defn memoized
"Like `memoize` but takes an explicit cache atom (possibly nil) and
immediately applies memoized f to given arguments."
[cache f & args]
(if-not cache
(apply f args)
@ -73,6 +73,9 @@
(swap! cache assoc args dv)
@dv))))
(comment (memoized nil +)
(memoized nil + 5 12))
(defn ba-concat ^bytes [^bytes ba1 ^bytes ba2]
(let [s1 (alength ba1)
s2 (alength ba2)

5
test/test_nippy/main.clj
View file

@ -7,9 +7,8 @@
(def test-data (dissoc nippy/stress-data :bytes))
(def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes))
(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password "secret")
#(nippy/freeze-to-bytes % :password "secret")))
(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"])
#(nippy/freeze-to-bytes % :password [:cached "secret"])))
(deftest test-roundtrip-defaults (is (= test-data (roundtrip-defaults test-data))))
(deftest test-roundtrip-encrypted (is (= test-data (roundtrip-encrypted test-data))))