luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev'

Browse source
This commit is contained in:
Peter Taoussanis 2013-06-16 19:36:57 +07:00
commit 439ad90381
7 changed files with 73 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -2,7 +2,7 @@ Current [semantic](http://semver.org/) version:
```clojure
[com.taoensso/nippy "1.2.1"] ; Stable
[com.taoensso/nippy "2.0.0-alpha6"] ; Development (notes below)
[com.taoensso/nippy "2.0.0-alpha7"] ; Development (notes below)
```
2.x adds pluggable compression, crypto support (also pluggable), an improved API (including much better error messages), and hugely improved performance. It **is backwards compatible**, but please note that the old `freeze-to-bytes`/`thaw-from-bytes` API has been **deprecated** in favor of `freeze`/`thaw`. **PLEASE REPORT ANY PROBLEMS!**

BIN
benchmarks/chart.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 22 KiB

2
project.clj
View file

@ -1,4 +1,4 @@
(defproject com.taoensso/nippy "2.0.0-alpha6"
(defproject com.taoensso/nippy "2.0.0-alpha7"
:description "Clojure serialization library"
:url "https://github.com/ptaoussanis/nippy"
:license {:name "Eclipse Public License"

125
src/taoensso/nippy.clj
View file

@ -4,8 +4,8 @@
{:author "Peter Taoussanis"}
(:require [taoensso.nippy
(utils :as utils)
(compression :as compression)
(encryption :as encryption)])
(compression :as compression :refer (snappy-compressor))
(encryption :as encryption :refer (aes128-encryptor))])
(:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream
ByteArrayInputStream]
[clojure.lang Keyword BigInt Ratio PersistentQueue PersistentTreeMap
@ -13,7 +13,6 @@
IPersistentSet IPersistentCollection]))
;;;; Nippy 2.x+ header spec (4 bytes)
(def ^:private ^:const head-version 1)
(def ^:private head-sig (.getBytes "NPY" "UTF-8"))
(def ^:private head-meta "Final byte stores version-dependent metadata."
@ -162,13 +161,16 @@
(comment (wrap-header (.getBytes "foo") {:compressed? true
:encrypted? false}))
(declare assert-legacy-args)
(defn freeze
"Serializes arg (any Clojure data type) to a byte array. Set :legacy-mode to
true to produce bytes readble by Nippy < 2.x."
^bytes [x & [{:keys [print-dup? password compressor encryptor legacy-mode]
:or {print-dup? true
compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]]
compressor snappy-compressor
encryptor aes128-encryptor}}]]
(when legacy-mode (assert-legacy-args compressor password))
(let [ba (ByteArrayOutputStream.)
stream (DataOutputStream. ba)]
(binding [*print-dup* print-dup?] (freeze-to-stream x stream))
@ -258,88 +260,56 @@
(when (utils/ba= head-sig* head-sig)
[data-ba (head-meta meta-id {:unrecognized-header? true})]))))
(defn throw-thaw-ex [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
(defn thaw
"Deserializes frozen bytes to their original Clojure data type.
:legacy-mode options:
false - Nippy >= 2.x data only (best).
true - Nippy < 2.x data only (deprecated).
:auto - Mixed data (default, migrating).
In most cases you'll want :auto if you're using a preexisting data set, and
`false` otherwise.
"Deserializes frozen bytes to their original Clojure data type. Supports data
frozen with current and all previous versions of Nippy.
WARNING: Enabling `:read-eval?` can lead to security vulnerabilities unless
you are sure you know what you're doing."
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-mode
strict?]
:or {legacy-mode :auto
compressor compression/default-snappy-compressor
encryptor encryption/default-aes128-encryptor}}]]
[^bytes ba & [{:keys [read-eval? password compressor encryptor legacy-opts]
:or {legacy-opts {:compressed? true}
compressor snappy-compressor
encryptor aes128-encryptor}}]]
(let [try-thaw-data
(fn [data-ba {decompress? :compressed? decrypt? :encrypted?
:or {decompress? compressor
decrypt? password}
:as head-meta}]
(let [apparent-header? (not (empty? head-meta))]
(let [ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
try-thaw-data
(fn [data-ba {:keys [compressed? encrypted?] :as head-meta}]
(let [password (when encrypted? password) ; => also head-meta
compressor (if head-meta
(when compressed? compressor)
(when (:compressed? legacy-opts) snappy-compressor))]
(try
(let [ba data-ba
ba (if decrypt? (encryption/decrypt encryptor password ba) ba)
ba (if decompress? (compression/decompress compressor ba) ba)
ba (if password (encryption/decrypt encryptor password ba) ba)
ba (if compressor (compression/decompress compressor ba) ba)
stream (DataInputStream. (ByteArrayInputStream. ba))]
(binding [*read-eval* read-eval?] (thaw-from-stream stream)))
(catch Exception e
(cond
decrypt? (throw-thaw-ex "Wrong password/encryptor?" e)
decompress? (throw-thaw-ex "Encrypted data or wrong compressor?" e)
:else
(if apparent-header?
(throw-thaw-ex "Corrupt data?" e)
(throw-thaw-ex "Encrypted and/or compressed data?" e)))))))]
password (ex "Wrong password/encryptor?" e)
compressor (if head-meta (ex "Encrypted data or wrong compressor?" e)
(ex "Uncompressed data?" e))
:else (if head-meta (ex "Corrupt data?" e)
(ex "Compressed data?" e)))))))]
(if (= legacy-mode true)
(try-thaw-data ba nil)
(if-let [[data-ba {:keys [unrecognized-header? compressed? encrypted?]
:as head-meta}] (try-parse-header ba)]
(if (= legacy-mode :auto)
(try
;; Header seems okay, but we won't trust its metadata for
;; error-reporting purposes
(try-thaw-data data-ba head-meta)
(catch Exception _ (try-thaw-data ba nil)))
(cond ; Trust metadata, give fancy error messages
unrecognized-header?
(throw-thaw-ex
"Unrecognized header. Data frozen with newer Nippy version?")
(and strict? (not encrypted?) password)
(throw-thaw-ex (str "Unencrypted data. Try again w/o password.\n"
"Disable `:strict?` option to ignore this error. "))
(and strict? (not compressed?) compressor)
(throw-thaw-ex (str "Uncompressed data. Try again w/o compressor.\n"
"Disable `:strict?` option to ignore this error."))
(cond ; Header appears okay
(and (not legacy-opts) unrecognized-header?) ; Conservative
(ex "Unrecognized header. Data frozen with newer Nippy version?")
(and compressed? (not compressor))
(throw-thaw-ex "Compressed data. Try again with compressor.")
(ex "Compressed data. Try again with compressor.")
(and encrypted? (not password))
(throw-thaw-ex "Encrypted data. Try again with password.")
:else (try-thaw-data data-ba head-meta)))
(ex "Encrypted data. Try again with password.")
:else (try (try-thaw-data data-ba head-meta)
(catch Exception _ (try-thaw-data ba nil))))
;; Header definitely not okay
(if (= legacy-mode :auto)
(try-thaw-data ba nil) ; Legacy thaw
(throw-thaw-ex
(str "Not Nippy data, data frozen with Nippy < 2.x, "
"or corrupt data?\n"
"See `:legacy-mode` option for data frozen with Nippy < 2.x.")))))))
(try-thaw-data ba nil))))
(comment (thaw (freeze "hello"))
(thaw (freeze "hello" {:compressor nil}))
(thaw (freeze "hello" {:compressor nil}) {:legacy-mode false
:strict? true}) ; ex
(thaw (freeze "hello" {:password [:salted "p"]})) ; ex
(thaw (freeze "hello") {:password [:salted "p"]}))
@ -393,19 +363,24 @@
;;;; Deprecated API
(defn- assert-legacy-args [compressor password]
(when password
(throw (AssertionError. "Encryption not supported in legacy mode.")))
(when (and compressor (not= compressor snappy-compressor))
(throw (AssertionError. "Only Snappy compressor supported in legacy mode."))))
(defn freeze-to-bytes "DEPRECATED: Use `freeze` instead."
^bytes [x & {:keys [print-dup? compress? password]
^bytes [x & {:keys [print-dup? compress?]
:or {print-dup? true
compress? true}}]
(freeze x {:print-dup? print-dup?
:compressor (when compress? compression/default-snappy-compressor)
:password password
:legacy-mode true}))
(freeze x {:legacy-mode true
:print-dup? print-dup?
:compressor (when compress? snappy-compressor)
:password nil}))
(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead."
[ba & {:keys [read-eval? compressed? password]
[ba & {:keys [read-eval? compressed?]
:or {compressed? true}}]
(thaw ba {:read-eval? read-eval?
:compressor (when compressed? compression/default-snappy-compressor)
:password password
:legacy-mode true}))
(thaw ba {:legacy-opts {:compressed? compressed?}
:read-eval? read-eval?
:password nil}))

7
src/taoensso/nippy/compression.clj
View file

@ -11,11 +11,10 @@
;;;; Default implementations
(deftype DefaultSnappyCompressor []
(deftype SnappyCompressor []
ICompressor
(compress [_ ba] (org.iq80.snappy.Snappy/compress ba))
(decompress [_ ba] (org.iq80.snappy.Snappy/uncompress ba 0 (alength ^bytes ba))))
(def default-snappy-compressor
"Default org.iq80.snappy.Snappy compressor."
(DefaultSnappyCompressor.))
(def snappy-compressor "Default org.iq80.snappy.Snappy compressor."
(SnappyCompressor.))

16
src/taoensso/nippy/encryption.clj
View file

@ -13,9 +13,6 @@
;;;; Default digests, ciphers, etc.
(def ^:private ^:const aes128-block-size (int 16))
(def ^:private ^:const salt-size (int 16))
(def ^:private ^javax.crypto.Cipher aes128-cipher
(javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))
(def ^:private ^java.security.MessageDigest sha512-md
@ -23,6 +20,9 @@
(def ^:private ^java.security.SecureRandom prng
(java.security.SecureRandom/getInstance "SHA1PRNG"))
(def ^:private ^:const aes128-block-size (.getBlockSize aes128-cipher))
(def ^:private ^:const salt-size aes128-block-size)
(defn- rand-bytes [size] (let [seed (byte-array size)] (.nextBytes prng seed) seed))
;;;; Default keygen
@ -51,7 +51,7 @@
(defn- destructure-typed-pwd
[typed-password]
(letfn [(throw-ex []
(throw (Exception.
(throw (AssertionError.
(str "Expected password form: "
"[<#{:salted :cached}> <password-string>].\n "
"See `default-aes128-encryptor` docstring for details!"))))]
@ -64,7 +64,7 @@
(comment (destructure-typed-pwd [:salted "foo"]))
(defrecord DefaultAES128Encryptor [key-cache]
(defrecord AES128Encryptor [key-cache]
IEncryptor
(encrypt [this typed-pwd data-ba]
(let [[type pwd] (destructure-typed-pwd typed-pwd)
@ -93,7 +93,7 @@
^javax.crypto.spec.SecretKeySpec key iv)
(.doFinal aes128-cipher data-ba))))
(def default-aes128-encryptor
(def aes128-encryptor
"Alpha - subject to change.
Default 128bit AES encryptor with multi-round SHA-512 keygen.
@ -129,12 +129,12 @@
Faster than `aes128-salted`, and harder to attack any particular key - but
increased danger if a key is somehow compromised."
(DefaultAES128Encryptor. (atom {})))
(AES128Encryptor. (atom {})))
;;;; Default implementation
(comment
(def dae default-aes128-encryptor)
(def dae aes128-encryptor)
(def secret-ba (.getBytes "Secret message" "UTF-8"))
(encrypt dae "p" secret-ba) ; Malformed
(time (encrypt dae [:salted "p"] secret-ba))

26
test/taoensso/nippy/tests/main.clj
View file

@ -6,13 +6,17 @@
;; Remove stuff from stress-data that breaks roundtrip equality
(def test-data (dissoc nippy/stress-data :bytes))
;;;; Basic data integrity
(expect test-data ((comp thaw freeze) test-data))
(expect test-data ((comp thaw #(freeze % {:legacy-mode true})) test-data))
(expect test-data ((comp #(thaw % {:password [:salted "p"]})
#(freeze % {:password [:salted "p"]}))
test-data))
(expect AssertionError (thaw (freeze test-data {:password "malformed"})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})
{:compressor nil}))
(expect ; Snappy lib compatibility (for legacy versions of Nippy)
(let [^bytes raw-ba (freeze test-data {:compressor nil})
^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
@ -23,24 +27,4 @@
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
;;;; API stuff
;;; Strict/auto mode - compression
(expect test-data (thaw (freeze test-data {:compressor nil})))
(expect Exception (thaw (freeze test-data {:compressor nil})
{:legacy-mode false
:strict? true}))
;;; Strict/auto mode - encryption
(expect test-data (thaw (freeze test-data) {:password [:salted "p"]}))
(expect Exception (thaw (freeze test-data) {:password [:salted "p"]
:legacy-mode false
:strict? true}))
;;; Encryption - passwords
(expect Exception (thaw (freeze test-data {:password "malformed"})))
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
(expect test-data (thaw (freeze test-data {:password [:salted "p"]})
{:password [:salted "p"]}))
(expect (benchmarks/autobench)) ; Also tests :cached passwords