Crypto: simplify design, add auto salting

Have decided to simplify the API even further and bring configuration down to
essentially one decision: do you want auto salting, or key caching?

README.md

@@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version:
 
 ```clojure
 [com.taoensso/nippy "1.2.1"]        ; Stable
-[com.taoensso/nippy "1.3.0-alpha1"] ; Development (adds crypto support)
+[com.taoensso/nippy "1.3.0-alpha2"] ; Development (adds crypto support)
 ```
 
 # Nippy, a Clojure serialization library
@@ -18,7 +18,7 @@ Nippy is an attempt to provide a drop-in, high-performance alternative to the re
   * **Reader-fallback** for difficult/future types (including Clojure 1.4+ tagged literals).
   * **Full test coverage** for every supported type.
   * [Snappy](http://code.google.com/p/snappy/) **integrated de/compression** for efficient storage and network transfer.
-  * Enable **high-strength encryption** with a single `:password "my-password"` option. (1.3.0+)
+  * Enable **high-strength encryption** with a single `:password [:salted "my-password"]` option. (1.3.0+)
 
 ## Getting started
 

project.clj

@@ -1,4 +1,4 @@
-(defproject com.taoensso/nippy "1.3.0-alpha1"
+(defproject com.taoensso/nippy "1.3.0-alpha2"
   :description "Clojure serialization library"
   :url "https://github.com/ptaoussanis/nippy"
   :license {:name "Eclipse Public License"

src/taoensso/nippy.clj

@@ -167,16 +167,15 @@
 
 (defn freeze-to-bytes
   "Serializes x to a byte array and returns the array."
-  ^bytes [x & {:keys [crypto compress? print-dup? salt password]
+  ^bytes [x & {:keys [compress? print-dup? password]
-               :or   {crypto     crypto/crypto-default
+               :or   {compress?  true
-                      compress?  true
                       print-dup? true}}]
   (let [ba     (ByteArrayOutputStream.)
         stream (DataOutputStream. ba)]
     (freeze-to-stream! stream x print-dup?)
     (let [ba (.toByteArray ba)
           ba (if compress? (utils/compress-bytes ba) ba)
-          ba (if password  (crypto/encrypt crypto salt password ba) ba)]
+          ba (if password  (crypto/encrypt-aes128 password ba) ba)]
       ba)))
 
 ;;;; Thawing
@@ -255,11 +254,11 @@
 
 (defn thaw-from-bytes
   "Deserializes an object from given byte array."
-  [ba & {:keys [crypto compressed? read-eval? salt password]
+  [ba & {:keys [compressed? read-eval? password]
-         :or   {crypto      crypto/crypto-default
+         :or   {compressed? true
                 read-eval?  false ; For `read-string` injection safety - NB!!!
-                compressed? true}}]
+                }}]
-  (-> (let [ba (if password    (crypto/decrypt crypto salt password ba) ba)
+  (-> (let [ba (if password    (crypto/decrypt-aes128 password ba) ba)
             ba (if compressed? (utils/uncompress-bytes ba) ba)]
         ba)
       (ByteArrayInputStream.)

src/taoensso/nippy/benchmarks.clj

@@ -13,11 +13,9 @@
 (defn reader-thaw   [x] (binding [*read-eval* false] (read-string x)))
 (def  reader-roundtrip (comp reader-thaw reader-freeze))
 
-(def crypto-opts [:password "secret" :crypto crypto/crypto-default-cached])
-
 (def roundtrip-defaults  (comp nippy/thaw-from-bytes nippy/freeze-to-bytes))
-(def roundtrip-encrypted (comp #(apply nippy/thaw-from-bytes % crypto-opts)
+(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "p"])
-                               #(apply nippy/freeze-to-bytes % crypto-opts)))
+                               #(nippy/freeze-to-bytes % :password [:cached "p"])))
 (def roundtrip-fast      (comp #(nippy/thaw-from-bytes % :compressed? false)
                                #(nippy/freeze-to-bytes % :compress?   false)))
 
@@ -48,11 +46,11 @@
 
     (println
      {:encrypted
-      {:freeze (bench (apply freeze-to-bytes data crypto-opts))
+      {:freeze (bench (freeze-to-bytes data :password [:cached "p"]))
-       :thaw   (let [frozen (apply freeze-to-bytes data crypto-opts)]
+       :thaw   (let [frozen (freeze-to-bytes data :password [:cached "p"])]
-                 (bench (apply thaw-from-bytes frozen crypto-opts)))
+                 (bench (thaw-from-bytes frozen :password [:cached "p"])))
        :round  (bench (roundtrip-encrypted data))
-       :data-size (count (apply freeze-to-bytes data crypto-opts))}})
+       :data-size (count (freeze-to-bytes data :password [:cached "p"]))}})
 
     (println
      {:fast
@@ -66,9 +64,9 @@
 
   ;;; 11 June 2013: Clojure 1.5.1, Nippy 1.3.0-alpha1
   ;; {:reader    {:freeze 17042, :thaw 31579, :round 48379, :data-size 22954}}
+  ;; {:fast      {:freeze 3078,  :thaw 4684,  :round 8117,  :data-size 13274}}
   ;; {:defaults  {:freeze 3810,  :thaw 5295,  :round 9052,  :data-size 12394}}
   ;; {:encrypted {:freeze 5800,  :thaw 6862,  :round 12317, :data-size 12416}}
-  ;; {:fast      {:freeze 3078,  :thaw 4684,  :round 8117,  :data-size 13274}}
 
   ;;; Clojure 1.5.1, Nippy 1.2.1 (+ sorted-set, sorted-map)
   ;; (def data (dissoc data :sorted-set :sorted-map))

src/taoensso/nippy/crypto.clj

@@ -3,113 +3,145 @@
   Simple no-nonsense crypto with reasonable defaults. Because your Clojure data
   deserves some privacy."
   {:author "Peter Taoussanis"}
-  (:require [taoensso.nippy.utils :as utils]))
+  (:require [clojure.string       :as str]
+            [taoensso.nippy.utils :as utils]))
 
-(defprotocol ICrypto "Simple cryptography interface."
+;;;; Interface
-  (gen-key ^javax.crypto.spec.SecretKeySpec [crypto salt pwd]
-    "Returns an appropriate SecretKeySpec.")
-  (encrypt ^bytes [crypto salt pwd ba] "Returns encrypted bytes.")
-  (decrypt ^bytes [crypto salt pwd ba] "Returns decrypted bytes."))
 
-(defrecord CryptoAES [cipher-type default-salt key-gen-opts cache])
+(defprotocol IEncrypter
+  (gen-key ^javax.crypto.spec.SecretKeySpec [encrypter salt-ba pwd])
+  (encrypt ^bytes [encrypter pwd ba])
+  (decrypt ^bytes [encrypter pwd ba]))
 
-(def ^:private ^java.security.MessageDigest sha-md
+(defrecord AES128Encrypter [key-work-factor key-cache])
-  (java.security.MessageDigest/getInstance "SHA-512"))
 
+;;;; Digests, ciphers, etc.
+
+;; 128bit keys have good JVM availability and are
+;; entirely sufficient, Ref. http://goo.gl/2YRQG
 (def ^:private ^:const aes128-block-size (int 16))
+(def ^:private ^:const salt-size         (int 16))
+
+(def ^:private ^javax.crypto.Cipher aes128-cipher
+  (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))
+(def ^:private ^java.security.MessageDigest sha512-md
+  (java.security.MessageDigest/getInstance "SHA-512"))
+(def ^:private ^java.security.SecureRandom prng
+  (java.security.SecureRandom/getInstance "SHA1PRNG"))
+
+(defn- rand-bytes [size] (let [seed (byte-array size)] (.nextBytes prng seed) seed))
+
+;;;; Default keygen
 
 (defn- sha512-key
-  "Default SHA512-based key generator. Good JVM availability without extra
+  "SHA512-based key generator. Good JVM availability without extra dependencies
-  dependencies (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple
+  (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds."
-  rounds. VERY aggressive multiples (>64) possible+recommended when cached."
+  [salt-ba ^String pwd key-work-factor]
-  [^String salted-pwd & [{:keys [rounds-multiple]
+  (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")]
-                          :or   {rounds-multiple 5}}]] ; Cacheable
+                     (if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba))
-  (loop [^bytes ba (.getBytes salted-pwd "UTF-8")
+         n (* (int Short/MAX_VALUE) key-work-factor)]
-         n (* (int Short/MAX_VALUE) (or rounds-multiple 5))]
     (if-not (zero? n)
-      (recur (.digest sha-md ba) (dec n))
+      (recur (.digest sha512-md ba) (dec n))
-      (-> ba
+      (-> ba (java.util.Arrays/copyOf aes128-block-size)
-          ;; 128bit keys have good JVM availability and are
-          ;; entirely sufficient, Ref. http://goo.gl/2YRQG
-          (java.util.Arrays/copyOf aes128-block-size)
              (javax.crypto.spec.SecretKeySpec. "AES")))))
 
 (comment
-  (time (sha512-key "hi" {:rounds-multiple 1}))   ; ~40ms per hash (fast)
+  (time (sha512-key nil "hi" 1))   ; ~40ms per hash (fast)
-  (time (sha512-key "hi" {:rounds-multiple 5}))   ; ~180ms  (default)
+  (time (sha512-key nil "hi" 5))   ; ~180ms  (default)
-  (time (sha512-key "hi" {:rounds-multiple 32}))  ; ~1200ms (conservative)
+  (time (sha512-key nil "hi" 32))  ; ~1200ms (conservative)
-  (time (sha512-key "hi" {:rounds-multiple 128})) ; ~4500ms (paranoid)
+  (time (sha512-key nil "hi" 128)) ; ~4500ms (paranoid)
   )
 
-(def ^:private cipher* (memoize #(javax.crypto.Cipher/getInstance %)))
+;;;; Default implementation
-(defn- cipher ^javax.crypto.Cipher [cipher-type] (cipher* cipher-type))
 
-(def ^:private ^java.security.SecureRandom rand-gen
+(extend-type AES128Encrypter
-  (java.security.SecureRandom/getInstance "SHA1PRNG"))
+  IEncrypter
-(defn- rand-bytes [size] (let [seed (make-array Byte/TYPE size)]
+  (gen-key [{:keys [key-work-factor key-cache]} salt-ba pwd]
-                           (.nextBytes rand-gen seed) seed))
+    ;; Trade-off: salt-ba and key-cache mutually exclusive
+    (utils/memoized key-cache sha512-key salt-ba pwd key-work-factor))
 
-(extend-type CryptoAES
+  (encrypt [{:keys [key-cache] :as this} pwd data-ba]
-  ICrypto
+    (let [salt?     (not key-cache)
-  (gen-key [{:keys [default-salt key-gen-opts cache]} salt pwd]
-    (utils/apply-memoized cache
-     sha512-key (str (or salt default-salt) pwd) key-gen-opts))
-
-  (encrypt [{:keys [cipher-type cache] :as crypto} salt pwd ba]
-    (let [cipher (cipher cipher-type)
-          key    (gen-key crypto salt pwd)
           iv-ba     (rand-bytes aes128-block-size)
+          salt-ba   (when salt? (rand-bytes salt-size))
+          prefix-ba (if-not salt? iv-ba (utils/ba-concat iv-ba salt-ba))
+          key       (gen-key this salt-ba pwd)
           iv        (javax.crypto.spec.IvParameterSpec. iv-ba)]
-      (.init cipher javax.crypto.Cipher/ENCRYPT_MODE key iv)
+      (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE key iv)
-      (utils/ba-concat iv-ba (.doFinal cipher ba))))
+      (utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba))))
 
-  (decrypt [{:keys [cipher-type cache] :as crypto} salt pwd ba]
+  (decrypt [{:keys [key-cache] :as this} pwd ba]
-    (let [cipher (cipher cipher-type)
+    (let [salt?       (not key-cache)
-          key    (gen-key crypto salt pwd)
+          prefix-size (+ aes128-block-size (if salt? salt-size 0))
-          [iv-ba data-ba] (utils/ba-split ba aes128-block-size)
+          [prefix-ba data-ba] (utils/ba-split ba prefix-size)
+          [iv-ba salt-ba]     (if-not salt? [prefix-ba nil]
+                                (utils/ba-split prefix-ba aes128-block-size))
+          key (gen-key this salt-ba pwd)
           iv  (javax.crypto.spec.IvParameterSpec. iv-ba)]
-      (.init cipher javax.crypto.Cipher/DECRYPT_MODE key iv)
+      (.init    aes128-cipher javax.crypto.Cipher/DECRYPT_MODE key iv)
-      (.doFinal cipher data-ba))))
+      (.doFinal aes128-cipher data-ba))))
 
-(defn crypto-aes128
+(def aes128-salted
-  "Returns a new CryptoAES object with options:
+  "USE CASE: You want more than a small, finite number of passwords (e.g. each
-    :default-salt    - Shared fallback password salt when none is provided. If
+             item encrypted will use a unique user-provided password).
-                       the use case allows it, a unique random salt per
-                       encrypted item is better.
-    :cache-keys?     - IMPORTANT. DO enable this if and ONLY if your use case
-                       involves only a small, finite number of unique secret
-                       keys (salt+password)s. Dramatically improves `gen-key`
-                       performance in those cases and (as a result) allows for
-                       a *much* stronger `key-work-factor`.
-    :key-work-factor - O(n) CPU time needed to generate keys. Larger factors
-                       provide more protection against brute-force attacks but
-                       make encryption+decryption slower if `:cache-keys?` is
-                       not enabled.
 
-                       Some sensible values (from fast to strong):
+  IMPLEMENTATION: Uses a relatively cheap key hash, but automatically salts
-                         Without caching: 1, 5,  10
+                  every key.
-                            With caching: 5, 32, 64, 128
 
-  See also `crypto-default` and `crypto-default-cached` for sensible ready-made
+  PROS: Each key is independent so would need to be attacked independently.
-  CryptoAES objects."
+  CONS: Key caching impossible, so there's an inherent trade-off between
-  [& [{:keys [default-salt cache-keys? key-work-factor]
+        encryption/decryption speed and the difficulty of attacking any
-       :or   {default-salt "XA~I3(:]3'ck5!M[z\\m`l^0mltR~y/]Arq_d9+$`e#yJssN^8"
+        particular key.
-              key-work-factor 5}}]]
-  (CryptoAES. "AES/CBC/PKCS5Padding"
-              default-salt
-              {:rounds-multiple (int key-work-factor)}
-              (when cache-keys? (atom {}))))
 
-(def crypto-default        (crypto-aes128))
+  Slower than `aes128-cached`, and easier to attack any particular key."
-(def crypto-default-cached (crypto-aes128 {:cache-keys?     true
+  (AES128Encrypter. 5 nil))
-                                           :key-work-factor 64}))
+
+(def aes128-cached
+  "USE CASE: You want only a small, finite number of passwords (e.g. a limited
+             number of staff/admins, or you'll be using a single password to
+             encrypt many items).
+
+  IMPLEMENTATION: Uses a _very_ expensive (but cached) key hash, and no salt.
+
+  PROS: Great amortized encryption/decryption speed. Expensive key hash makes
+        attacking any particular key very difficult.
+  CONS: Using a small number of keys for many encrypted items means that if any
+        key _is_ somehow compromised, _all_ items encrypted with that key are
+        compromised.
+
+  Faster than `aes128-salted`, and harder to attack any particular key - but
+  increased danger if a key is somehow compromised."
+  (AES128Encrypter. 64 (atom {})))
+
+(defn- destructure-typed-password
+  "[<type> <password>] -> [Encrypter <password>]"
+  [typed-password]
+  (letfn [(throw-ex []
+            (throw (Exception.
+              (str "Expected password form: "
+                   "[<#{:salted :cached}> <password-string>].\n "
+                   "See `aes128-salted`, `aes128-cached` for details."))))]
+    (if-not (vector? typed-password)
+      (throw-ex)
+      (let [[type password] typed-password]
+        [(case type :salted aes128-salted :cached aes128-cached (throw-ex))
+         password]))))
+
+(defn encrypt-aes128 [typed-password ba]
+  (let [[encrypter password] (destructure-typed-password typed-password)]
+    (encrypt encrypter password ba)))
+
+(defn decrypt-aes128 [typed-password ba]
+  (let [[encrypter password] (destructure-typed-password typed-password)]
+    (decrypt encrypter password ba)))
 
 (comment
-  (time (gen-key crypto-default        "my-salt" "my-password"))
+  (encrypt-aes128 "my-password" (.getBytes "Secret message")) ; Malformed
-  (time (gen-key crypto-default-cached "my-salt" "my-password"))
+  (time (gen-key aes128-salted nil "my-password"))
+  (time (gen-key aes128-cached nil "my-password"))
   (time (->> (.getBytes "Secret message" "UTF-8")
-             (encrypt crypto-default "s" "p")
+             (encrypt-aes128 [:salted "p"])
-             (encrypt crypto-default "s" "p")
+             (encrypt-aes128 [:cached "p"])
-             (decrypt crypto-default "s" "p")
+             (decrypt-aes128 [:cached "p"])
-             (decrypt crypto-default "s" "p")
+             (decrypt-aes128 [:salted "p"])
              (String.))))

src/taoensso/nippy/utils.clj

@@ -61,9 +61,9 @@
 (defn compress-bytes   [^bytes ba] (Snappy/compress   ba))
 (defn uncompress-bytes [^bytes ba] (Snappy/uncompress ba 0 (alength ba)))
 
-(defn apply-memoized
+(defn memoized
-  "A cross between `memoize` and `apply`. Operates like `apply` but accepts an
+  "Like `memoize` but takes an explicit cache atom (possibly nil) and
-  optional {<args> <value> ...} cache atom."
+  immediately applies memoized f to given arguments."
   [cache f & args]
   (if-not cache
     (apply f args)
@@ -73,6 +73,9 @@
         (swap! cache assoc args dv)
         @dv))))
 
+(comment (memoized nil +)
+         (memoized nil + 5 12))
+
 (defn ba-concat ^bytes [^bytes ba1 ^bytes ba2]
   (let [s1  (alength ba1)
         s2  (alength ba2)

test/test_nippy/main.clj

@@ -7,9 +7,8 @@
 (def test-data (dissoc nippy/stress-data :bytes))
 
 (def roundtrip-defaults  (comp nippy/thaw-from-bytes nippy/freeze-to-bytes))
-(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password "secret")
+(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"])
-                               #(nippy/freeze-to-bytes % :password "secret")))
+                               #(nippy/freeze-to-bytes % :password [:cached "secret"])))
-
 (deftest test-roundtrip-defaults  (is (= test-data (roundtrip-defaults  test-data))))
 (deftest test-roundtrip-encrypted (is (= test-data (roundtrip-encrypted test-data))))