diff --git a/README.md b/README.md index 1860bd1..42429cc 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version: ```clojure [com.taoensso/nippy "1.2.1"] ; Stable -[com.taoensso/nippy "1.3.0-alpha1"] ; Development (adds crypto support) +[com.taoensso/nippy "1.3.0-alpha2"] ; Development (adds crypto support) ``` # Nippy, a Clojure serialization library @@ -18,7 +18,7 @@ Nippy is an attempt to provide a drop-in, high-performance alternative to the re * **Reader-fallback** for difficult/future types (including Clojure 1.4+ tagged literals). * **Full test coverage** for every supported type. * [Snappy](http://code.google.com/p/snappy/) **integrated de/compression** for efficient storage and network transfer. - * Enable **high-strength encryption** with a single `:password "my-password"` option. (1.3.0+) + * Enable **high-strength encryption** with a single `:password [:salted "my-password"]` option. (1.3.0+) ## Getting started diff --git a/project.clj b/project.clj index 12f82bb..601a9bf 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "1.3.0-alpha1" +(defproject com.taoensso/nippy "1.3.0-alpha2" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License" diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 6df0e03..12c294a 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -167,16 +167,15 @@ (defn freeze-to-bytes "Serializes x to a byte array and returns the array." - ^bytes [x & {:keys [crypto compress? print-dup? salt password] - :or {crypto crypto/crypto-default - compress? true + ^bytes [x & {:keys [compress? print-dup? password] + :or {compress? true print-dup? true}}] (let [ba (ByteArrayOutputStream.) stream (DataOutputStream. ba)] (freeze-to-stream! stream x print-dup?) (let [ba (.toByteArray ba) ba (if compress? (utils/compress-bytes ba) ba) - ba (if password (crypto/encrypt crypto salt password ba) ba)] + ba (if password (crypto/encrypt-aes128 password ba) ba)] ba))) ;;;; Thawing @@ -255,11 +254,11 @@ (defn thaw-from-bytes "Deserializes an object from given byte array." - [ba & {:keys [crypto compressed? read-eval? salt password] - :or {crypto crypto/crypto-default + [ba & {:keys [compressed? read-eval? password] + :or {compressed? true read-eval? false ; For `read-string` injection safety - NB!!! - compressed? true}}] - (-> (let [ba (if password (crypto/decrypt crypto salt password ba) ba) + }}] + (-> (let [ba (if password (crypto/decrypt-aes128 password ba) ba) ba (if compressed? (utils/uncompress-bytes ba) ba)] ba) (ByteArrayInputStream.) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 03f9c2e..902c5c1 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -13,11 +13,9 @@ (defn reader-thaw [x] (binding [*read-eval* false] (read-string x))) (def reader-roundtrip (comp reader-thaw reader-freeze)) -(def crypto-opts [:password "secret" :crypto crypto/crypto-default-cached]) - (def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(def roundtrip-encrypted (comp #(apply nippy/thaw-from-bytes % crypto-opts) - #(apply nippy/freeze-to-bytes % crypto-opts))) +(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "p"]) + #(nippy/freeze-to-bytes % :password [:cached "p"]))) (def roundtrip-fast (comp #(nippy/thaw-from-bytes % :compressed? false) #(nippy/freeze-to-bytes % :compress? false))) @@ -48,11 +46,11 @@ (println {:encrypted - {:freeze (bench (apply freeze-to-bytes data crypto-opts)) - :thaw (let [frozen (apply freeze-to-bytes data crypto-opts)] - (bench (apply thaw-from-bytes frozen crypto-opts))) + {:freeze (bench (freeze-to-bytes data :password [:cached "p"])) + :thaw (let [frozen (freeze-to-bytes data :password [:cached "p"])] + (bench (thaw-from-bytes frozen :password [:cached "p"]))) :round (bench (roundtrip-encrypted data)) - :data-size (count (apply freeze-to-bytes data crypto-opts))}}) + :data-size (count (freeze-to-bytes data :password [:cached "p"]))}}) (println {:fast @@ -66,9 +64,9 @@ ;;; 11 June 2013: Clojure 1.5.1, Nippy 1.3.0-alpha1 ;; {:reader {:freeze 17042, :thaw 31579, :round 48379, :data-size 22954}} + ;; {:fast {:freeze 3078, :thaw 4684, :round 8117, :data-size 13274}} ;; {:defaults {:freeze 3810, :thaw 5295, :round 9052, :data-size 12394}} ;; {:encrypted {:freeze 5800, :thaw 6862, :round 12317, :data-size 12416}} - ;; {:fast {:freeze 3078, :thaw 4684, :round 8117, :data-size 13274}} ;;; Clojure 1.5.1, Nippy 1.2.1 (+ sorted-set, sorted-map) ;; (def data (dissoc data :sorted-set :sorted-map)) diff --git a/src/taoensso/nippy/crypto.clj b/src/taoensso/nippy/crypto.clj index 619ef02..fdad5b3 100644 --- a/src/taoensso/nippy/crypto.clj +++ b/src/taoensso/nippy/crypto.clj @@ -3,113 +3,145 @@ Simple no-nonsense crypto with reasonable defaults. Because your Clojure data deserves some privacy." {:author "Peter Taoussanis"} - (:require [taoensso.nippy.utils :as utils])) + (:require [clojure.string :as str] + [taoensso.nippy.utils :as utils])) -(defprotocol ICrypto "Simple cryptography interface." - (gen-key ^javax.crypto.spec.SecretKeySpec [crypto salt pwd] - "Returns an appropriate SecretKeySpec.") - (encrypt ^bytes [crypto salt pwd ba] "Returns encrypted bytes.") - (decrypt ^bytes [crypto salt pwd ba] "Returns decrypted bytes.")) +;;;; Interface -(defrecord CryptoAES [cipher-type default-salt key-gen-opts cache]) +(defprotocol IEncrypter + (gen-key ^javax.crypto.spec.SecretKeySpec [encrypter salt-ba pwd]) + (encrypt ^bytes [encrypter pwd ba]) + (decrypt ^bytes [encrypter pwd ba])) -(def ^:private ^java.security.MessageDigest sha-md - (java.security.MessageDigest/getInstance "SHA-512")) +(defrecord AES128Encrypter [key-work-factor key-cache]) +;;;; Digests, ciphers, etc. + +;; 128bit keys have good JVM availability and are +;; entirely sufficient, Ref. http://goo.gl/2YRQG (def ^:private ^:const aes128-block-size (int 16)) +(def ^:private ^:const salt-size (int 16)) + +(def ^:private ^javax.crypto.Cipher aes128-cipher + (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding")) +(def ^:private ^java.security.MessageDigest sha512-md + (java.security.MessageDigest/getInstance "SHA-512")) +(def ^:private ^java.security.SecureRandom prng + (java.security.SecureRandom/getInstance "SHA1PRNG")) + +(defn- rand-bytes [size] (let [seed (byte-array size)] (.nextBytes prng seed) seed)) + +;;;; Default keygen (defn- sha512-key - "Default SHA512-based key generator. Good JVM availability without extra - dependencies (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple - rounds. VERY aggressive multiples (>64) possible+recommended when cached." - [^String salted-pwd & [{:keys [rounds-multiple] - :or {rounds-multiple 5}}]] ; Cacheable - (loop [^bytes ba (.getBytes salted-pwd "UTF-8") - n (* (int Short/MAX_VALUE) (or rounds-multiple 5))] + "SHA512-based key generator. Good JVM availability without extra dependencies + (PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds." + [salt-ba ^String pwd key-work-factor] + (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] + (if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba)) + n (* (int Short/MAX_VALUE) key-work-factor)] (if-not (zero? n) - (recur (.digest sha-md ba) (dec n)) - (-> ba - ;; 128bit keys have good JVM availability and are - ;; entirely sufficient, Ref. http://goo.gl/2YRQG - (java.util.Arrays/copyOf aes128-block-size) - (javax.crypto.spec.SecretKeySpec. "AES"))))) + (recur (.digest sha512-md ba) (dec n)) + (-> ba (java.util.Arrays/copyOf aes128-block-size) + (javax.crypto.spec.SecretKeySpec. "AES"))))) (comment - (time (sha512-key "hi" {:rounds-multiple 1})) ; ~40ms per hash (fast) - (time (sha512-key "hi" {:rounds-multiple 5})) ; ~180ms (default) - (time (sha512-key "hi" {:rounds-multiple 32})) ; ~1200ms (conservative) - (time (sha512-key "hi" {:rounds-multiple 128})) ; ~4500ms (paranoid) + (time (sha512-key nil "hi" 1)) ; ~40ms per hash (fast) + (time (sha512-key nil "hi" 5)) ; ~180ms (default) + (time (sha512-key nil "hi" 32)) ; ~1200ms (conservative) + (time (sha512-key nil "hi" 128)) ; ~4500ms (paranoid) ) -(def ^:private cipher* (memoize #(javax.crypto.Cipher/getInstance %))) -(defn- cipher ^javax.crypto.Cipher [cipher-type] (cipher* cipher-type)) +;;;; Default implementation -(def ^:private ^java.security.SecureRandom rand-gen - (java.security.SecureRandom/getInstance "SHA1PRNG")) -(defn- rand-bytes [size] (let [seed (make-array Byte/TYPE size)] - (.nextBytes rand-gen seed) seed)) +(extend-type AES128Encrypter + IEncrypter + (gen-key [{:keys [key-work-factor key-cache]} salt-ba pwd] + ;; Trade-off: salt-ba and key-cache mutually exclusive + (utils/memoized key-cache sha512-key salt-ba pwd key-work-factor)) -(extend-type CryptoAES - ICrypto - (gen-key [{:keys [default-salt key-gen-opts cache]} salt pwd] - (utils/apply-memoized cache - sha512-key (str (or salt default-salt) pwd) key-gen-opts)) + (encrypt [{:keys [key-cache] :as this} pwd data-ba] + (let [salt? (not key-cache) + iv-ba (rand-bytes aes128-block-size) + salt-ba (when salt? (rand-bytes salt-size)) + prefix-ba (if-not salt? iv-ba (utils/ba-concat iv-ba salt-ba)) + key (gen-key this salt-ba pwd) + iv (javax.crypto.spec.IvParameterSpec. iv-ba)] + (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE key iv) + (utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba)))) - (encrypt [{:keys [cipher-type cache] :as crypto} salt pwd ba] - (let [cipher (cipher cipher-type) - key (gen-key crypto salt pwd) - iv-ba (rand-bytes aes128-block-size) - iv (javax.crypto.spec.IvParameterSpec. iv-ba)] - (.init cipher javax.crypto.Cipher/ENCRYPT_MODE key iv) - (utils/ba-concat iv-ba (.doFinal cipher ba)))) + (decrypt [{:keys [key-cache] :as this} pwd ba] + (let [salt? (not key-cache) + prefix-size (+ aes128-block-size (if salt? salt-size 0)) + [prefix-ba data-ba] (utils/ba-split ba prefix-size) + [iv-ba salt-ba] (if-not salt? [prefix-ba nil] + (utils/ba-split prefix-ba aes128-block-size)) + key (gen-key this salt-ba pwd) + iv (javax.crypto.spec.IvParameterSpec. iv-ba)] + (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE key iv) + (.doFinal aes128-cipher data-ba)))) - (decrypt [{:keys [cipher-type cache] :as crypto} salt pwd ba] - (let [cipher (cipher cipher-type) - key (gen-key crypto salt pwd) - [iv-ba data-ba] (utils/ba-split ba aes128-block-size) - iv (javax.crypto.spec.IvParameterSpec. iv-ba)] - (.init cipher javax.crypto.Cipher/DECRYPT_MODE key iv) - (.doFinal cipher data-ba)))) +(def aes128-salted + "USE CASE: You want more than a small, finite number of passwords (e.g. each + item encrypted will use a unique user-provided password). -(defn crypto-aes128 - "Returns a new CryptoAES object with options: - :default-salt - Shared fallback password salt when none is provided. If - the use case allows it, a unique random salt per - encrypted item is better. - :cache-keys? - IMPORTANT. DO enable this if and ONLY if your use case - involves only a small, finite number of unique secret - keys (salt+password)s. Dramatically improves `gen-key` - performance in those cases and (as a result) allows for - a *much* stronger `key-work-factor`. - :key-work-factor - O(n) CPU time needed to generate keys. Larger factors - provide more protection against brute-force attacks but - make encryption+decryption slower if `:cache-keys?` is - not enabled. + IMPLEMENTATION: Uses a relatively cheap key hash, but automatically salts + every key. - Some sensible values (from fast to strong): - Without caching: 1, 5, 10 - With caching: 5, 32, 64, 128 + PROS: Each key is independent so would need to be attacked independently. + CONS: Key caching impossible, so there's an inherent trade-off between + encryption/decryption speed and the difficulty of attacking any + particular key. - See also `crypto-default` and `crypto-default-cached` for sensible ready-made - CryptoAES objects." - [& [{:keys [default-salt cache-keys? key-work-factor] - :or {default-salt "XA~I3(:]3'ck5!M[z\\m`l^0mltR~y/]Arq_d9+$`e#yJssN^8" - key-work-factor 5}}]] - (CryptoAES. "AES/CBC/PKCS5Padding" - default-salt - {:rounds-multiple (int key-work-factor)} - (when cache-keys? (atom {})))) + Slower than `aes128-cached`, and easier to attack any particular key." + (AES128Encrypter. 5 nil)) -(def crypto-default (crypto-aes128)) -(def crypto-default-cached (crypto-aes128 {:cache-keys? true - :key-work-factor 64})) +(def aes128-cached + "USE CASE: You want only a small, finite number of passwords (e.g. a limited + number of staff/admins, or you'll be using a single password to + encrypt many items). + + IMPLEMENTATION: Uses a _very_ expensive (but cached) key hash, and no salt. + + PROS: Great amortized encryption/decryption speed. Expensive key hash makes + attacking any particular key very difficult. + CONS: Using a small number of keys for many encrypted items means that if any + key _is_ somehow compromised, _all_ items encrypted with that key are + compromised. + + Faster than `aes128-salted`, and harder to attack any particular key - but + increased danger if a key is somehow compromised." + (AES128Encrypter. 64 (atom {}))) + +(defn- destructure-typed-password + "[ ] -> [Encrypter ]" + [typed-password] + (letfn [(throw-ex [] + (throw (Exception. + (str "Expected password form: " + "[<#{:salted :cached}> ].\n " + "See `aes128-salted`, `aes128-cached` for details."))))] + (if-not (vector? typed-password) + (throw-ex) + (let [[type password] typed-password] + [(case type :salted aes128-salted :cached aes128-cached (throw-ex)) + password])))) + +(defn encrypt-aes128 [typed-password ba] + (let [[encrypter password] (destructure-typed-password typed-password)] + (encrypt encrypter password ba))) + +(defn decrypt-aes128 [typed-password ba] + (let [[encrypter password] (destructure-typed-password typed-password)] + (decrypt encrypter password ba))) (comment - (time (gen-key crypto-default "my-salt" "my-password")) - (time (gen-key crypto-default-cached "my-salt" "my-password")) + (encrypt-aes128 "my-password" (.getBytes "Secret message")) ; Malformed + (time (gen-key aes128-salted nil "my-password")) + (time (gen-key aes128-cached nil "my-password")) (time (->> (.getBytes "Secret message" "UTF-8") - (encrypt crypto-default "s" "p") - (encrypt crypto-default "s" "p") - (decrypt crypto-default "s" "p") - (decrypt crypto-default "s" "p") + (encrypt-aes128 [:salted "p"]) + (encrypt-aes128 [:cached "p"]) + (decrypt-aes128 [:cached "p"]) + (decrypt-aes128 [:salted "p"]) (String.)))) \ No newline at end of file diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index e3d90ff..a02f2e5 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -61,9 +61,9 @@ (defn compress-bytes [^bytes ba] (Snappy/compress ba)) (defn uncompress-bytes [^bytes ba] (Snappy/uncompress ba 0 (alength ba))) -(defn apply-memoized - "A cross between `memoize` and `apply`. Operates like `apply` but accepts an - optional { ...} cache atom." +(defn memoized + "Like `memoize` but takes an explicit cache atom (possibly nil) and + immediately applies memoized f to given arguments." [cache f & args] (if-not cache (apply f args) @@ -73,6 +73,9 @@ (swap! cache assoc args dv) @dv)))) +(comment (memoized nil +) + (memoized nil + 5 12)) + (defn ba-concat ^bytes [^bytes ba1 ^bytes ba2] (let [s1 (alength ba1) s2 (alength ba2) diff --git a/test/test_nippy/main.clj b/test/test_nippy/main.clj index 7a7eb59..30a11d1 100644 --- a/test/test_nippy/main.clj +++ b/test/test_nippy/main.clj @@ -7,9 +7,8 @@ (def test-data (dissoc nippy/stress-data :bytes)) (def roundtrip-defaults (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password "secret") - #(nippy/freeze-to-bytes % :password "secret"))) - +(def roundtrip-encrypted (comp #(nippy/thaw-from-bytes % :password [:cached "secret"]) + #(nippy/freeze-to-bytes % :password [:cached "secret"]))) (deftest test-roundtrip-defaults (is (= test-data (roundtrip-defaults test-data)))) (deftest test-roundtrip-encrypted (is (= test-data (roundtrip-encrypted test-data))))