luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev'

Browse source
This commit is contained in:
Peter Taoussanis 2013-06-16 19:36:57 +07:00
commit 439ad90381
7 changed files with 73 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version:
```clojure
[com.taoensso/nippy "1.2.1"] ; Stable
[com.taoensso/nippy "2.0.0-alpha6"] ; Development (notes below)
[com.taoensso/nippy "2.0.0-alpha7"] ; Development (notes below)
```
2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the old `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`. **PLEASE REPORT ANY PROBLEMS!**

BIN
benchmarks/chart.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 22 KiB

2
project.clj
View file

@ -1,4 +1,4 @@
(defproject com.taoensso/nippy "2.0.0-alpha6"
(defproject com.taoensso/nippy "2.0.0-alpha7"
:description "Clojure serialization library"
:url "https://github.com/ptaoussanis/nippy"
:license {:name "Eclipse Public License"

135
src/taoensso/nippy.clj
View file

@ -4,8 +4,8 @@
{:author "Peter Taoussanis"}
(:require [taoensso.nippy
(utils :as utils)
(compression :as compression)
(encryption :as encryption)])
(compression :as compression :refer (snappy-compressor))
(encryption :as encryption :refer (aes128-encryptor))])
(:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream
ByteArrayInputStream]
[clojure.lang Keyword BigInt Ratio PersistentQueue PersistentTreeMap
@ -13,7 +13,6 @@
IPersistentSet IPersistentCollection]))
;;;; Nippy 2.x+ header spec (4 bytes)
(def ^:private ^:const head-version 1)
(def ^:private head-sig (.getBytes "NPY" "UTF-8"))
(def ^:private head-meta "Final byte stores version-dependent metadata."
@ -162,13 +161,16 @@
(comment (wrap-header (.getBytes "foo") {:compressed? true
:encrypted? false}))
(declare assert-legacy-args)
(defn freeze
"Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to
true to produce bytes readble by Nippy < 2.x."
^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode]
:or {print-dup? true
compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]]
compressor snappy-compressor
encryptor aes128-encryptor}}]]
(when legacy-mode (assert-legacy-args compressor password))
(let [ba (ByteArrayOutputStream.)
stream (DataOutputStream. ba)]
(binding [*print-dup* print-dup?] (freeze-to-stream x stream))
@ -258,88 +260,56 @@
(when (utils/ba= head-sig* head-sig)
[data-ba (head-meta meta-id {:unrecognized-header? true})]))))
(defn throw-thaw-ex [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
(defn thaw
"Deserializes frozen bytes to their original Clojure data type.
:legacy-mode options:
false - Nippy >= 2.x data only (best).
true - Nippy < 2.x data only (deprecated).
:auto - Mixed data (default, migrating).
In most cases you'll want :auto if you're using a preexisting data set, and
`false` otherwise.
"Deserializes frozen bytes to their original Clojure data type. Supports data
frozen with current and all previous versions of Nippy.
WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless
you are sure you know what you're doing."
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode
strict?]
:or {legacy-mode :auto
compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]]
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-opts]
:or {legacy-opts {:compressed? true}
compressor snappy-compressor
encryptor aes128-encryptor}}]]
(let [try-thaw-data
(fn [data-ba {decompress? :compressed? decrypt? :encrypted?
:or {decompress? compressor
decrypt? password}
:as head-meta}]
(let [apparent-header? (not (empty? head-meta))]
(let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
try-thaw-data
(fn [data-ba {:keys [compressed? encrypted?] :as head-meta}]
(let [password (when encrypted? password) ; => also head-meta
compressor (if head-meta
(when compressed? compressor)
(when (:compressed? legacy-opts) snappy-compressor))]
(try
(let [ba data-ba
ba (if decrypt? (encryption/decrypt encryptor password ba) ba)
ba (if decompress? (compression/decompress compressor ba) ba)
ba (if password (encryption/decrypt encryptor password ba) ba)
ba (if compressor (compression/decompress compressor ba) ba)
stream (DataInputStream. (ByteArrayInputStream. ba))]
(binding [*read-eval* read-eval?] (thaw-from-stream stream)))
(catch Exception e
(cond
decrypt? (throw-thaw-ex "Wrong password/encryptor?" e)
decompress? (throw-thaw-ex "Encrypted data or wrong compressor?" e)
:else
(if apparent-header?
(throw-thaw-ex "Corrupt data?" e)
(throw-thaw-ex "Encrypted and/or compressed data?" e)))))))]
password (ex "Wrong password/encryptor?" e)
compressor (if head-meta (ex "Encrypted data or wrong compressor?" e)
(ex "Uncompressed data?" e))
:else (if head-meta (ex "Corrupt data?" e)
(ex "Compressed data?" e)))))))]
(if (= legacy-mode true)
(try-thaw-data ba nil)
(if-let [[data-ba {:keys [unrecognized-header? compressed? encrypted?]
:as head-meta}] (try-parse-header ba)]
(if (= legacy-mode :auto)
(try
;; Header seems okay, but we won't trust its metadata for
;; error-reporting purposes
(try-thaw-data data-ba head-meta)
(catch Exception _ (try-thaw-data ba nil)))
(if-let [[data-ba {:keys [unrecognized-header? compressed? encrypted?]
:as head-meta}] (try-parse-header ba)]
(cond ; Trust metadata, give fancy error messages
unrecognized-header?
(throw-thaw-ex
"Unrecognized header. Data frozen with newer Nippy version?")
(and strict? (not encrypted?) password)
(throw-thaw-ex (str "Unencrypted data. Try again w/o password.\n"
"Disable `:strict?` option to ignore this error. "))
(and strict? (not compressed?) compressor)
(throw-thaw-ex (str "Uncompressed data. Try again w/o compressor.\n"
"Disable `:strict?` option to ignore this error."))
(and compressed? (not compressor))
(throw-thaw-ex "Compressed data. Try again with compressor.")
(and encrypted? (not password))
(throw-thaw-ex "Encrypted data. Try again with password.")
:else (try-thaw-data data-ba head-meta)))
(cond ; Header appears okay
(and (not legacy-opts) unrecognized-header?) ; Conservative
(ex "Unrecognized header. Data frozen with newer Nippy version?")
(and compressed? (not compressor))
(ex "Compressed data. Try again with compressor.")
(and encrypted? (not password))
(ex "Encrypted data. Try again with password.")
:else (try (try-thaw-data data-ba head-meta)
(catch Exception _ (try-thaw-data ba nil))))
;; Header definitely not okay
(if (= legacy-mode :auto)
(try-thaw-data ba nil) ; Legacy thaw
(throw-thaw-ex
(str "Not Nippy data, data frozen with Nippy < 2.x, "
"or corrupt data?\n"
"See `:legacy-mode` option for data frozen with Nippy < 2.x.")))))))
;; Header definitely not okay
(try-thaw-data ba nil))))
(comment (thaw (freeze "hello"))
(thaw (freeze "hello" {:compressor nil}))
(thaw (freeze "hello" {:compressor nil}) {:legacy-mode false
:strict? true}) ; ex
(thaw (freeze "hello" {:password [:salted "p"]})) ; ex
(thaw (freeze "hello") {:password [:salted "p"]}))
@ -393,19 +363,24 @@
;;;; Deprecated API
(defn- assert-legacy-args [compressor password]
(when password
(throw (AssertionError. "Encryption not supported in legacy mode.")))
(when (and compressor (not= compressor snappy-compressor))
(throw (AssertionError. "Only Snappy compressor supported in legacy mode."))))
(defn freeze-to-bytes "DEPRECATED: Use `freeze` instead."
^bytes [x & {:keys [print-dup? compress? password]
^bytes [x & {:keys [print-dup? compress?]
:or {print-dup? true
compress? true}}]
(freeze x {:print-dup? print-dup?
:compressor (when compress? compression/default-snappy-compressor)
:password password
:legacy-mode true}))
(freeze x {:legacy-mode true
:print-dup? print-dup?
:compressor (when compress? snappy-compressor)
:password nil}))
(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead."
[ba & {:keys [read-eval? compressed? password]
[ba & {:keys [read-eval? compressed?]
:or {compressed? true}}]
(thaw ba {:read-eval? read-eval?
:compressor (when compressed? compression/default-snappy-compressor)
:password password
:legacy-mode true}))
(thaw ba {:legacy-opts {:compressed? compressed?}
:read-eval? read-eval?
:password nil}))

7
src/taoensso/nippy/compression.clj
View file

@ -11,11 +11,10 @@
;;;; Default implementations
(deftype DefaultSnappyCompressor []
(deftype SnappyCompressor []
ICompressor
(compress [_ ba] (org.iq80.snappy.Snappy/compress ba))
(decompress [_ ba] (org.iq80.snappy.Snappy/uncompress ba 0 (alength ^bytes ba))))
(def default-snappy-compressor
"Default org.iq80.snappy.Snappy compressor."
(DefaultSnappyCompressor.))
(def snappy-compressor "Default org.iq80.snappy.Snappy compressor."
(SnappyCompressor.))

16
src/taoensso/nippy/encryption.clj
View file

@ -13,9 +13,6 @@
;;;; Default digests, ciphers, etc.
(def ^:private ^:const aes128-block-size (int 16))
(def ^:private ^:const salt-size (int 16))
(def ^:private ^javax.crypto.Cipher aes128-cipher
(javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))
(def ^:private ^java.security.MessageDigest sha512-md
@ -23,6 +20,9 @@
(def ^:private ^java.security.SecureRandom prng
(java.security.SecureRandom/getInstance "SHA1PRNG"))
(def ^:private ^:const aes128-block-size (.getBlockSize aes128-cipher))
(def ^:private ^:const salt-size aes128-block-size)
(defn- rand-bytes [size] (let [seed (byte-array size)] (.nextBytes prng seed) seed))
;;;; Default keygen
@ -51,7 +51,7 @@
(defn- destructure-typed-pwd
[typed-password]
(letfn [(throw-ex []
(throw (Exception.
(throw (AssertionError.
(str "Expected password form: "
"[<#{:salted :cached}> <password-string>].\n "
"See `default-aes128-encryptor` docstring for details!"))))]
@ -64,7 +64,7 @@
(comment (destructure-typed-pwd [:salted "foo"]))
(defrecord DefaultAES128Encryptor [key-cache]
(defrecord AES128Encryptor [key-cache]
IEncryptor
(encrypt [this typed-pwd data-ba]
(let [[type pwd] (destructure-typed-pwd typed-pwd)
@ -93,7 +93,7 @@
^javax.crypto.spec.SecretKeySpec key iv)
(.doFinal aes128-cipher data-ba))))
(def default-aes128-encryptor
(def aes128-encryptor
"Alpha - subject to change.
Default 128bit AES encryptor with multi-round SHA-512 keygen.
@ -129,12 +129,12 @@
Faster than `aes128-salted`, and harder to attack any particular key - but
increased danger if a key is somehow compromised."
(DefaultAES128Encryptor. (atom {})))
(AES128Encryptor. (atom {})))
;;;; Default implementation
(comment
(def dae default-aes128-encryptor)
(def dae aes128-encryptor)
(def secret-ba (.getBytes "Secret message" "UTF-8"))
(encrypt dae "p" secret-ba) ; Malformed
(time (encrypt dae [:salted "p"] secret-ba))

26
test/taoensso/nippy/tests/main.clj
View file

@ -6,13 +6,17 @@
;; Remove stuff from stress-data that breaks roundtrip equality
(def test-data (dissoc nippy/stress-data :bytes))
;;;; Basic data integrity
(expect test-data ((comp thaw freeze) test-data))
(expect test-data ((comp thaw #(freeze % {:legacy-mode true})) test-data))
(expect test-data ((comp #(thaw % {:password [:salted "p"]})
#(freeze % {:password [:salted "p"]}))
test-data))
(expect AssertionError (thaw (freeze test-data {:password "malformed"})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})
{:compressor nil}))
(expect ; Snappy lib compatibility (for legacy versions of Nippy)
(let [^bytes raw-ba (freeze test-data {:compressor nil})
^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
@ -23,24 +27,4 @@
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
;;;; API stuff
;;; Strict/auto mode - compression
(expect test-data (thaw (freeze test-data {:compressor nil})))
(expect Exception (thaw (freeze test-data {:compressor nil})
{:legacy-mode false
:strict? true}))
;;; Strict/auto mode - encryption
(expect test-data (thaw (freeze test-data) {:password [:salted "p"]}))
(expect Exception (thaw (freeze test-data) {:password [:salted "p"]
:legacy-mode false
:strict? true}))
;;; Encryption - passwords
(expect Exception (thaw (freeze test-data {:password "malformed"})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
(expect test-data (thaw (freeze test-data {:password [:salted "p"]})
{:password [:salted "p"]}))
(expect (benchmarks/autobench)) ; Also tests :cached passwords