diff --git a/README.md b/README.md index 40c06d6..24227c8 100644 --- a/README.md +++ b/README.md @@ -115,9 +115,9 @@ Couldn't be simpler! See also the lower-level `freeze-to-out!` and `thaw-from-in!` fns for operating on `DataOutput` and `DataInput` types directly. -### Encryption (currently in **ALPHA**) +### Encryption (v2+) -Nippy v2+ also gives you **dead simple data encryption**. Add a single option to your usual freeze/thaw calls like so: +Nippy also gives you **dead simple data encryption**. Add a single option to your usual freeze/thaw calls like so: ```clojure (nippy/freeze nippy/stress-data {:password [:salted "my-password"]}) ; Encrypt @@ -126,7 +126,7 @@ Nippy v2+ also gives you **dead simple data encryption**. Add a single option to There's two default forms of encryption on offer: `:salted` and `:cached`. Each of these makes carefully-chosen trade-offs and is suited to one of two common use cases. See the `aes128-encryptor` [docstring](http://ptaoussanis.github.io/nippy/taoensso.nippy.encryption.html) for a detailed explanation of why/when you'd want one or the other. -### Custom types (v2.1+, ALPHA - subject to change) +### Custom types (v2.1+) ```clojure (defrecord MyType [data]) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index a83992d..8ee57c3 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -6,9 +6,8 @@ [taoensso.encore :as encore] [taoensso.nippy (utils :as utils) - (compression :as compression :refer (lz4-compressor - snappy-compressor)) - (encryption :as encryption :refer (aes128-encryptor))]) + (compression :as compression) + (encryption :as encryption)]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream DataOutputStream Serializable ObjectOutputStream ObjectInputStream DataOutput DataInput] @@ -21,25 +20,32 @@ IRecord ISeq])) ;;;; Nippy 2.x+ header spec (4 bytes) -;; Header is optional but recommended + enabled by default. Uses: +;; Header is optional but recommended + enabled by default. Purpose: ;; * Sanity check (data appears to be Nippy data). ;; * Nippy version check (=> supports changes to data schema over time). -;; * Encrypted &/or compressed data identification. +;; * Supports :auto thaw compressor, encryptor. ;; -(def ^:private ^:const head-version 2) +(def ^:private ^:const head-version 1) (def ^:private head-sig (.getBytes "NPY" "UTF-8")) (def ^:private ^:const head-meta "Final byte stores version-dependent metadata." - {;;; Nippy <= v2.6 with Snappy as default compressor - (byte 0) {:version 1 :compressed? false :encrypted? false} - (byte 1) {:version 1 :compressed? true :encrypted? false} - (byte 2) {:version 1 :compressed? false :encrypted? true} - (byte 3) {:version 1 :compressed? true :encrypted? true} - - ;;; EXPERIMENTAL: Nippy >= v2.7 with LZ4 as default compressor - (byte 4) {:version 2 :compressed? false :encrypted? false} - (byte 5) {:version 2 :compressed? true :encrypted? false} - (byte 6) {:version 2 :compressed? false :encrypted? true} - (byte 7) {:version 2 :compressed? true :encrypted? true}}) + {(byte 0) {:version 1 :compressor-id nil :encryptor-id nil} + (byte 4) {:version 1 :compressor-id nil :encryptor-id :else} + (byte 5) {:version 1 :compressor-id :else :encryptor-id nil} + (byte 6) {:version 1 :compressor-id :else :encryptor-id :else} + ;; + (byte 2) {:version 1 :compressor-id nil :encryptor-id :aes128-sha512} + ;; + (byte 1) {:version 1 :compressor-id :snappy :encryptor-id nil} + (byte 3) {:version 1 :compressor-id :snappy :encryptor-id :aes128-sha512} + (byte 7) {:version 1 :compressor-id :snappy :encryptor-id :else} + ;; + (byte 8) {:version 1 :compressor-id :lz4 :encryptor-id nil} + (byte 9) {:version 1 :compressor-id :lz4 :encryptor-id :aes128-sha512} + (byte 10) {:version 1 :compressor-id :lz4 :encryptor-id :else} + ;; + (byte 11) {:version 1 :compressor-id :lzma2 :encryptor-id nil} + (byte 12) {:version 1 :compressor-id :lzma2 :encryptor-id :aes128-sha512} + (byte 13) {:version 1 :compressor-id :lzma2 :encryptor-id :else}}) (defmacro when-debug-mode [& body] (when #_true false `(do ~@body))) @@ -86,7 +92,7 @@ (def ^:const id-ratio (int 70)) (def ^:const id-record (int 80)) - ;; (def ^:const id-type (int 81)) ; TODO + ;; (def ^:const id-type (int 81)) ; TODO? (def ^:const id-date (int 90)) (def ^:const id-uuid (int 91)) @@ -111,6 +117,21 @@ (def ^:const id-old-keyword (int 12)) ; as of 2.0.0-alpha5, for str consistecy ) +;;;; Ns imports (mostly for convenience of lib consumers) + +(encore/defalias compress compression/compress) +(encore/defalias decompress compression/decompress) +(encore/defalias snappy-compressor compression/snappy-compressor) +(encore/defalias lzma2-compressor compression/lzma2-compressor) +(encore/defalias lz4-compressor compression/lz4-compressor) +(encore/defalias lz4hc-compressor compression/lz4hc-compressor) + +(encore/defalias encrypt encryption/encrypt) +(encore/defalias decrypt encryption/decrypt) +(encore/defalias aes128-encryptor encryption/aes128-encryptor) + +(encore/defalias freezable? utils/freezable?) + ;;;; Freezing (defprotocol Freezable @@ -137,7 +158,7 @@ (let [x (with-meta x {:tag 'String})] `(write-bytes ~out (.getBytes ~x "UTF-8") ~small?))) -(defmacro write-compact-long "EXPERIMENTAL! Uses 2->9 bytes." [out x] +(defmacro write-compact-long "Uses 2->9 bytes." [out x] `(write-bytes ~out (.toByteArray (java.math.BigInteger/valueOf (long ~x))) :small)) @@ -304,19 +325,25 @@ :else ; Fallback #3: *final-freeze-fallback* (if-let [ffb *final-freeze-fallback*] (ffb x out) - (throw (Exception. (format "Unfreezable type: %s %s" - (type x) (str x)))))))) + (throw (ex-info (format "Unfreezable type: %s %s" (type x) (str x)) + {:type (type x) + :as-str (pr-str x)})))))) (def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta)) +(def ^:private get-head-ba + (memoize + (fn [head-meta] + (when-let [meta-id (get head-meta-id (assoc head-meta :version head-version))] + (encore/ba-concat head-sig (byte-array [meta-id])))))) -(defn- wrap-header [data-ba metadata] - (if-let [meta-id (head-meta-id (assoc metadata :version head-version))] - (let [head-ba (encore/ba-concat head-sig (byte-array [meta-id]))] - (encore/ba-concat head-ba data-ba)) - (throw (Exception. (str "Unrecognized header metadata: " metadata))))) +(defn- wrap-header [data-ba head-meta] + (if-let [head-ba (get-head-ba head-meta)] + (encore/ba-concat head-ba data-ba) + (throw (ex-info (format "Unrecognized header meta: %s" head-meta) + {:head-meta head-meta})))) -(comment (wrap-header (.getBytes "foo") {:compressed? true - :encrypted? false})) +(comment (wrap-header (.getBytes "foo") {:compressor-id :lz4 + :encryptor-id nil})) (defn freeze-to-out! "Low-level API. Serializes arg (any Clojure data type) to a DataOutput." @@ -324,27 +351,30 @@ (freeze-to-out data-output x)) (defn freeze - "Serializes arg (any Clojure data type) to a byte array. For custom types - extend the Clojure reader or see `extend-freeze`." - ^bytes [x & [{:keys [password compressor encryptor skip-header?] + "Serializes arg (any Clojure data type) to a byte array. To freeze custom + types, extend the Clojure reader or see `extend-freeze`." + ^bytes [x & [{:keys [compressor encryptor password skip-header?] :or {compressor lz4-compressor encryptor aes128-encryptor} :as opts}]] - (let [;;; Legacy mode is deprecated - compressor (if-not (:legacy-mode opts) compressor snappy-compressor) - encryptor (if-not (:legacy-mode opts) encryptor nil) - ;; - skip-header? (or skip-header? (:legacy-mode opts) ; Deprecated - ) - bas (ByteArrayOutputStream.) - sout (DataOutputStream. bas)] - (freeze-to-out! sout x) - (let [ba (.toByteArray bas) - ba (if compressor (compression/compress compressor ba) ba) - ba (if password (encryption/encrypt encryptor password ba) ba)] + (let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze + compressor (if-not legacy-mode? compressor snappy-compressor) + encryptor (when password (if-not legacy-mode? encryptor nil)) + skip-header? (or skip-header? legacy-mode?) + baos (ByteArrayOutputStream.) + dos (DataOutputStream. baos)] + (freeze-to-out! dos x) + (let [ba (.toByteArray baos) + ba (if-not compressor ba (compress compressor ba)) + ba (if-not encryptor ba (encrypt encryptor password ba))] (if skip-header? ba - (wrap-header ba {:compressed? (boolean compressor) - :encrypted? (boolean password)}))))) + (wrap-header ba + {:compressor-id (when-let [c compressor] + (or (compression/standard-header-ids + (compression/header-id c)) :else)) + :encryptor-id (when-let [e encryptor] + (or (encryption/standard-header-ids + (encryption/header-id e)) :else))}))))) ;;;; Thawing @@ -362,8 +392,7 @@ (defmacro read-utf8 [in & [small?]] `(String. (read-bytes ~in ~small?) "UTF-8")) -(defmacro read-compact-long "EXPERIMENTAL!" [in] - `(long (BigInteger. (read-bytes ~in :small)))) +(defmacro read-compact-long [in] `(long (BigInteger. (read-bytes ~in :small)))) (defmacro ^:private read-coll [in coll] `(let [in# ~in] (encore/repeatedly-into* ~coll (.readInt in#) (thaw-from-in in#)))) @@ -463,19 +492,25 @@ id-old-keyword (keyword (.readUTF in)) (if-not (neg? type-id) - (throw (Exception. (str "Unknown type ID: " type-id))) + (throw (ex-info (format "Unknown type ID: %s" type-id) + {:type-id type-id})) ;; Custom types (if-let [reader (get @custom-readers type-id)] (try (reader in) (catch Exception e - (throw (Exception. (str "Reader exception for custom type ID: " - (- type-id)) e)))) - (throw (Exception. (str "No reader provided for custom type ID: " - (- type-id))))))) + (throw (ex-info + (format "Reader exception for custom type ID: %s" + (- type-id)) + {:type-id (- type-id)} e)))) + (throw (ex-info + (format "No reader provided for custom type ID: %s" + (- type-id)) + {:type-id (- type-id)}))))) (catch Exception e - (throw (Exception. (format "Thaw failed against type-id: %s" type-id) e)))))) + (throw (ex-info (format "Thaw failed against type-id: %s" type-id) + {:type-id type-id} e)))))) (defn thaw-from-in! "Low-level API. Deserializes a frozen object from given DataInput to its @@ -486,121 +521,118 @@ (defn- try-parse-header [ba] (when-let [[head-ba data-ba] (encore/ba-split ba 4)] (let [[head-sig* [meta-id]] (encore/ba-split head-ba 3)] - (when (encore/ba= head-sig* head-sig) ; Appears to be well-formed - [data-ba (head-meta meta-id {:unrecognized-meta? true})])))) + (when (encore/ba= head-sig* head-sig) ; Header appears to be well-formed + [data-ba (get head-meta meta-id {:unrecognized-meta? true})])))) + +(defn- get-auto-compressor [compressor-id] + (case compressor-id + nil nil + :snappy snappy-compressor + :lzma2 lzma2-compressor + :lz4 lz4-compressor + :no-header (throw (ex-info ":auto not supported on headerless data." {})) + :else (throw (ex-info ":auto not supported for non-standard compressors." {})) + (throw (ex-info (format "Unrecognized :auto compressor id: %s" compressor-id) + {:compressor-id compressor-id})))) + +(defn- get-auto-encryptor [encryptor-id] + (case encryptor-id + nil nil + :aes128-sha512 aes128-encryptor + :no-header (throw (ex-info ":auto not supported on headerless data." {})) + :else (throw (ex-info ":auto not supported for non-standard encryptors.")) + (throw (ex-info (format "Unrecognized :auto encryptor id: %s" encryptor-id) + {:encryptor-id encryptor-id})))) (defn thaw "Deserializes a frozen object from given byte array to its original Clojure - data type. By default[1] supports data frozen with current and all previous - versions of Nippy. For custom types extend the Clojure reader or see - `extend-thaw`. + data type. Supports data frozen with current and all previous versions of + Nippy. To thaw custom types, extend the Clojure reader or see `extend-thaw`. - [1] :headerless-meta provides a fallback facility for data frozen without a - standard Nippy header (notably all Nippy v1 data). A default is provided for - Nippy v1 thaw compatibility, but it's recommended that you _disable_ this - fallback (`{:headerless-meta nil}`) if you're certain you won't be thawing - headerless data." - [^bytes ba & [{:keys [password compressor encryptor headerless-meta] + Options include: + :compressor - An ICompressor, :auto (requires Nippy header), or nil. + :encryptor - An IEncryptor, :auto (requires Nippy header), or nil." + [^bytes ba & [{:keys [compressor encryptor password] :or {compressor :auto - encryptor :auto - headerless-meta ; Recommend set to nil when possible - {:version 2 - :compressed? true - :encrypted? false}} + encryptor :auto} :as opts}]] - (let [headerless-meta (merge headerless-meta (:legacy-opts opts)) ; Deprecated - _ (assert (or (nil? headerless-meta) - (head-meta-id headerless-meta)) - "Bad :headerless-meta (should be nil or a valid `head-meta` value)") + (assert (not (contains? opts :headerless-meta)) + ":headerless-meta `thaw` option removed as of Nippy v2.7.") + + (let [ex (fn [msg & [e]] (throw (ex-info (format "Thaw failed: %s" msg) + {:opts (merge opts + {:compressor compressor + :encryptor encryptor})} + e))) + thaw-data + (fn [data-ba compressor-id encryptor-id] + (let [compressor (if-not (identical? compressor :auto) compressor + (get-auto-compressor compressor-id)) + encryptor (if-not (identical? encryptor :auto) encryptor + (get-auto-encryptor encryptor-id))] + + (when (and encryptor (not password)) + (ex "Password required for decryption.")) - ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e))) - try-thaw-data - (fn [data-ba {:keys [compressed? encrypted? version] - :as _head-or-headerless-meta}] - (let [encryptor (when (and password encrypted?) - (if-not (identical? encryptor :auto) encryptor - aes128-encryptor)) - compressor (when compressed? - (if-not (identical? compressor :auto) compressor - (case (int version) - 1 snappy-compressor - 2 lz4-compressor)))] (try (let [ba data-ba - ba (if encryptor (encryption/decrypt encryptor password ba) ba) - ba (if compressor (compression/decompress compressor ba) ba) - sin (DataInputStream. (ByteArrayInputStream. ba))] - (thaw-from-in! sin)) + ba (if-not encryptor ba (decrypt encryptor password ba)) + ba (if-not compressor ba (decompress compressor ba)) + dis (DataInputStream. (ByteArrayInputStream. ba))] + (thaw-from-in! dis)) (catch Exception e - (cond - encryptor (if head-meta (ex "Wrong password/encryptor?" e) - (ex "Unencrypted data?" e)) - compressor (if head-meta (ex "Encrypted data or wrong compressor?" e) - (ex "Uncompressed data?" e)) - :else (if head-meta (ex "Corrupt data?" e) - (ex "Data may be unfrozen, corrupt, compressed &/or encrypted.")))))))] + (ex "Decryption/decompression failure, or data unfrozen/damaged."))))) - (if-let [[data-ba {:keys [unrecognized-meta? compressed? encrypted?] + thaw-nippy-v1-data ; A little hackish, but necessary + (fn [data-ba] + (try (thaw-data data-ba :snappy nil) + (catch Exception _ + (thaw-data data-ba nil nil))))] + + (if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?] :as head-meta}] (try-parse-header ba)] - (cond ; A well-formed header _appears_ to be present - (and (not headerless-meta) ; Cautious. It's unlikely but possible the - ; header sig match was a fluke and not an - ; indication of a real, well-formed header. - ; May really be headerless. - unrecognized-meta?) - (ex "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?") - - ;;; It's still possible below that the header match was a fluke, but it's - ;;; _very_ unlikely. Therefore _not_ going to incl. - ;;; `(not headerless-meta)` conditions below. - - (and compressed? (not compressor)) - (ex "Compressed data? Try again with compressor.") - (and encrypted? (not password)) - (if (::tools-thaw? opts) ::need-password - (ex "Encrypted data? Try again with password.")) - :else (try (try-thaw-data data-ba head-meta) - (catch Exception e - (if headerless-meta - (try (try-thaw-data ba headerless-meta) - (catch Exception _ - (throw e))) - (throw e))))) + ;; A well-formed header _appears_ to be present (it's possible though + ;; unlikely that this is a fluke and data is actually headerless): + (try (thaw-data data-ba compressor-id encryptor-id) + (catch Exception e + (try (thaw-nippy-v1-data) + (catch Exception _ + (if unrecognized-meta? + (ex "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?" + e) + (throw e)))))) ;; Well-formed header definitely not present - (if headerless-meta - (try-thaw-data ba headerless-meta) - (ex "Data may be unfrozen, corrupt, compressed &/or encrypted."))))) + (try (thaw-nippy-v1-data ba) + (catch Exception _ + (thaw-data ba :no-header :no-header)))))) (comment (thaw (freeze "hello")) (thaw (freeze "hello" {:compressor nil})) - (thaw (freeze "hello" {:password [:salted "p"]})) ; ex + (thaw (freeze "hello" {:password [:salted "p"]})) ; ex: no pwd (thaw (freeze "hello") {:password [:salted "p"]})) ;;;; Custom types (defmacro extend-freeze - "Alpha - subject to change. - Extends Nippy to support freezing of a custom type (ideally concrete) with + "Extends Nippy to support freezing of a custom type (ideally concrete) with id ∈[1, 128]: (defrecord MyType [data]) (extend-freeze MyType 1 [x data-output] (.writeUTF [data-output] (:data x)))" [type custom-type-id [x out] & body] (assert (and (>= custom-type-id 1) (<= custom-type-id 128))) - `(extend-type ~type - Freezable + `(extend-type ~type Freezable (~'freeze-to-out* [~x ~(with-meta out {:tag 'java.io.DataOutput})] (write-id ~out ~(int (- custom-type-id))) ~@body))) (defonce custom-readers (atom {})) ; { (fn [data-input]) ...} (defmacro extend-thaw - "Alpha - subject to change. - Extends Nippy to support thawing of a custom type with id ∈[1, 128]: + "Extends Nippy to support thawing of a custom type with id ∈[1, 128]: (extend-thaw 1 [data-input] (->MyType (.readUTF data-input)))" [custom-type-id [in] & body] @@ -623,16 +655,14 @@ compress? (> ba-len 1024)] (.writeBoolean out compress?) (if-not compress? (write-bytes out ba) - (let [ba* (compression/compress compression/lzma2-compressor ba)] + (let [ba* (compress lzma2-compressor ba)] (write-bytes out ba*))))) (extend-thaw 128 [in] (let [compressed? (.readBoolean in) - ba (read-bytes in)] - (thaw ba {:compressor compression/lzma2-compressor - :headerless-meta {:version 2 - :compressed? compressed? - :encrypted? false}}))) + ba (read-bytes in)] + (thaw ba {:compressor (when compressed? lzma2-compressor) + :encryptor nil}))) (comment (->> (apply str (repeatedly 1000 rand)) @@ -719,8 +749,6 @@ ;;;; Tools -(encore/defalias freezable? utils/freezable?) - (defn inspect-ba "Alpha - subject to change." [ba & [thaw-opts]] (if-not (encore/bytes? ba) :not-ba @@ -734,15 +762,15 @@ [data-ba nippy-header] (or (try-parse-header unwrapped-ba) [unwrapped-ba :no-header])] - {:known-wrapper known-wrapper - :nippy2-header nippy-header ; Nippy v1.x didn't have a header - :thawable? (try (thaw unwrapped-ba thaw-opts) true - (catch Exception _ false)) - :unwrapped-ba unwrapped-ba - :data-ba data-ba - :unwrapped-size (alength ^bytes unwrapped-ba) - :ba-size (alength ^bytes ba) - :data-size (alength ^bytes data-ba)}))) + {:known-wrapper known-wrapper + :nippy-v2-header nippy-header ; Nippy v1.x didn't have a header + :thawable? (try (thaw unwrapped-ba thaw-opts) true + (catch Exception _ false)) + :unwrapped-ba unwrapped-ba + :data-ba data-ba + :unwrapped-size (alength ^bytes unwrapped-ba) + :ba-size (alength ^bytes ba) + :data-size (alength ^bytes data-ba)}))) (comment (inspect-ba (freeze "hello")) (seq (:data-ba (inspect-ba (freeze "hello"))))) @@ -754,17 +782,3 @@ (def thaw-from-stream! "DEPRECATED: Use `thaw-from-in!` instead." thaw-from-in!) - -(defn freeze-to-bytes "DEPRECATED: Use `freeze` instead." - ^bytes [x & {:keys [compress?] - :or {compress? true}}] - (freeze x {:skip-header? true - :compressor (when compress? snappy-compressor) - :password nil})) - -(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead." - [ba & {:keys [compressed?] - :or {compressed? true}}] - (thaw ba {:headerless-opts {:compressed? compressed?} - :compressor snappy-compressor - :password nil})) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 4ff2f47..e1abd51 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -3,8 +3,7 @@ (:require [clojure.tools.reader.edn :as edn] [clojure.data.fressian :as fressian] [taoensso.encore :as encore] - [taoensso.nippy :as nippy :refer (freeze thaw)] - [taoensso.nippy.compression :as compression])) + [taoensso.nippy :as nippy :refer (freeze thaw)])) (def data nippy/stress-data-benchable) @@ -45,16 +44,14 @@ #(thaw % {}))}) (println {:fast (bench1 #(freeze % {:compressor nil :skip-header? true}) - #(thaw % {:headerless-meta - {:version 2 - :compressed? false - :encrypted? false}}))}) + #(thaw % {:compressor nil + :encryptor nil}))}) (println {:encrypted (bench1 #(freeze % {:password [:cached "p"]}) #(thaw % {:password [:cached "p"]}))}) (when lzma2? ; Slow as molasses - (println {:lzma2 (bench1 #(freeze % {:compressor compression/lzma2-compressor}) - #(thaw % {:compressor compression/lzma2-compressor}))})) + (println {:lzma2 (bench1 #(freeze % {:compressor nippy/lzma2-compressor}) + #(thaw % {:compressor nippy/lzma2-compressor}))})) (when fressian? (println {:fressian (bench1 fressian-freeze fressian-thaw)}))) @@ -70,87 +67,21 @@ ;; (bench {:laps 2}) ;;; 2014 Apr 5 w/ headerless :fast, LZ4 replacing Snappy as default compressor - ;; {:default {:round 7669, :freeze 4157, :thaw 3512, :size 16143}} - ;; {:fast {:round 6918, :freeze 3636, :thaw 3282, :size 16992}} - ;; {:encrypted {:round 11814, :freeze 6180, :thaw 5634, :size 16164}} + {:default {:round 7669, :freeze 4157, :thaw 3512, :size 16143}} + {:fast {:round 6918, :freeze 3636, :thaw 3282, :size 16992}} + {:encrypted {:round 11814, :freeze 6180, :thaw 5634, :size 16164}} ;;; 2014 Jan 22: with common-type size optimizations, enlarged stress-data - ;; {:reader {:round 109544, :freeze 39523, :thaw 70021, :size 27681}} - ;; {:default {:round 9234, :freeze 5128, :thaw 4106, :size 15989}} - ;; {:fast {:round 7402, :freeze 4021, :thaw 3381, :size 16957}} - ;; {:encrypted {:round 12594, :freeze 6884, :thaw 5710, :size 16020}} - ;; {:lzma2 {:round 66759, :freeze 44246, :thaw 22513, :size 11208}} - ;; {:fressian {:round 13052, :freeze 8694, :thaw 4358, :size 16942}} + {:reader {:round 109544, :freeze 39523, :thaw 70021, :size 27681}} + {:default {:round 9234, :freeze 5128, :thaw 4106, :size 15989}} + {:fast {:round 7402, :freeze 4021, :thaw 3381, :size 16957}} + {:encrypted {:round 12594, :freeze 6884, :thaw 5710, :size 16020}} + {:lzma2 {:round 66759, :freeze 44246, :thaw 22513, :size 11208}} + {:fressian {:round 13052, :freeze 8694, :thaw 4358, :size 16942}} ;;; 19 Oct 2013: Nippy v2.3.0, with lzma2 & (nb!) round=freeze+thaw - ;; {:reader {:round 67798, :freeze 23202, :thaw 44596, :size 22971}} - ;; {:default {:round 3632, :freeze 2349, :thaw 1283, :size 12369}} - ;; {:encrypted {:round 6970, :freeze 4073, :thaw 2897, :size 12388}} - ;; {:fast {:round 3294, :freeze 2109, :thaw 1185, :size 13277}} - ;; {:lzma2 {:round 44590, :freeze 29567, :thaw 15023, :size 9076}} - - ;;; 11 Oct 2013: Nippy v2.2.0, with both ztellman mods - ;; {:defaults {:round 4319, :freeze 2950, :thaw 1446, :data-size 12369}} - ;; {:encrypted {:round 7675, :freeze 4479, :thaw 3160, :data-size 12388}} - ;; {:fast {:round 3928, :freeze 2530, :thaw 1269, :data-size 13277}} - ;; {:defaults-delta {:round 0.84 :freeze 0.79 :thaw 1.14}} ; vs 2.2.0 - - ;;; 11 Oct 2013: Nippy v2.2.0, with first ztellman mod - ;; {:defaults {:round 4059, :freeze 2578, :thaw 1351, :data-size 12342}} - ;; {:encrypted {:round 7248, :freeze 4058, :thaw 3041, :data-size 12372}} - ;; {:fast {:round 3430, :freeze 2085, :thaw 1229, :data-size 13277}} - ;; {:defaults-delta {:round 0.79 :freeze 0.69 :thaw 1.07}} ; vs 2.2.0 - - ;;; 11 Oct 2013: Nippy v2.2.0 - ;; {:defaults {:round 5135, :freeze 3711, :thaw 1266, :data-size 12393}} - ;; {:encrypted {:round 8655, :freeze 5323, :thaw 3036, :data-size 12420}} - ;; {:fast {:round 4670, :freeze 3282, :thaw 1294, :data-size 13277}} - - ;;; 7 Auguest 2013: Nippy v2.2.0-RC1 - ;; {:reader {:round 71582, :freeze 13656, :thaw 56730, :data-size 22964}} - ;; {:defaults {:round 5619, :freeze 3710, :thaw 1783, :data-size 12368}} - ;; {:encrypted {:round 9113, :freeze 5324, :thaw 3500, :data-size 12388}} - ;; {:fast {:round 5130, :freeze 3286, :thaw 1667, :data-size 13325}} - - ;;; 17 June 2013: Clojure 1.5.1, JVM 7 Nippy 2.0.0-alpha6 w/fast io-streams - ;; {:reader {:round 49819, :freeze 23601, :thaw 26247, :data-size 22966}} - ;; {:defaults {:round 5670, :freeze 3536, :thaw 1919, :data-size 12396}} - ;; {:encrypted {:round 9038, :freeze 5111, :thaw 3582, :data-size 12420}} - ;; {:fast {:round 5182, :freeze 3177, :thaw 1820, :data-size 13342}} - - ;;; 16 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha6 - ;; {:reader {:freeze 23601, :thaw 26247, :round 49819, :data-size 22966}} - ;; {:defaults {:freeze 3554, :thaw 2002, :round 5831, :data-size 12394}} - ;; {:encrypted {:freeze 5117, :thaw 3600, :round 9006, :data-size 12420}} - ;; {:fast {:freeze 3247, :thaw 1914, :round 5329, :data-size 13342}} - - ;;; 13 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha1 - ;; {:reader {:freeze 23124, :thaw 26469, :round 47674, :data-size 22923}} - ;; {:defaults {:freeze 4007, :thaw 2520, :round 6038, :data-size 12387}} - ;; {:encrypted {:freeze 5560, :thaw 3867, :round 9157, :data-size 12405}} - ;; {:fast {:freeze 3429, :thaw 2078, :round 5577, :data-size 13237}} - - ;;; 11 June 2013: Clojure 1.5.1, Nippy 1.3.0-alpha1 - ;; {:reader {:freeze 17042, :thaw 31579, :round 48379, :data-size 22954}} - ;; {:fast {:freeze 3078, :thaw 4684, :round 8117, :data-size 13274}} - ;; {:defaults {:freeze 3810, :thaw 5295, :round 9052, :data-size 12394}} - ;; {:encrypted {:freeze 5800, :thaw 6862, :round 12317, :data-size 12416}} - - ;;; Clojure 1.5.1, Nippy 1.2.1 (+ sorted-set, sorted-map) - ;; (def data (dissoc data :sorted-set :sorted-map)) - ;; {:reader {:freeze 15037, :thaw 27885, :round 43945}, - ;; :nippy {:freeze 3194, :thaw 4734, :round 8380}} - ;; {:reader-size 22975, :defaults-size 12400, :encrypted-size 12400} - - ;;; Clojure 1.4.0, Nippy 1.0.0 (+ tagged-uuid, tagged-date) - ;; {:reader {:freeze 22595, :thaw 31148, :round 54059} - ;; :nippy {:freeze 3324, :thaw 3725, :round 6918}} - - ;;; Clojure 1.3.0, Nippy 0.9.2 - ;; {:reader {:freeze 28505, :thaw 36451, :round 59545}, - ;; :nippy {:freeze 3751, :thaw 4184, :round 7769}} - - (println (bench* (roundtrip data))) ; Snappy implementations - ;; {:no-snappy [6163 6064 6042 6176] :JNI [6489 6446 6542 6412] - ;; :native-array-copy [6569 6419 6414 6590]} - ) + {:reader {:round 67798, :freeze 23202, :thaw 44596, :size 22971}} + {:default {:round 3632, :freeze 2349, :thaw 1283, :size 12369}} + {:encrypted {:round 6970, :freeze 4073, :thaw 2897, :size 12388}} + {:fast {:round 3294, :freeze 2109, :thaw 1185, :size 13277}} + {:lzma2 {:round 44590, :freeze 29567, :thaw 15023, :size 9076}}) diff --git a/src/taoensso/nippy/compression.clj b/src/taoensso/nippy/compression.clj index 85b7864..4f5b2eb 100644 --- a/src/taoensso/nippy/compression.clj +++ b/src/taoensso/nippy/compression.clj @@ -1,4 +1,4 @@ -(ns taoensso.nippy.compression "Alpha - subject to change." +(ns taoensso.nippy.compression {:author "Peter Taoussanis"} (:require [taoensso.encore :as encore]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream @@ -7,13 +7,17 @@ ;;;; Interface (defprotocol ICompressor + (header-id [compressor]) (compress ^bytes [compressor ba]) (decompress ^bytes [compressor ba])) ;;;; Default implementations +(def standard-header-ids "These'll support :auto thaw." #{:snappy :lzma2 :lz4}) + (deftype SnappyCompressor [] ICompressor + (header-id [_] :snappy) (compress [_ ba] (org.iq80.snappy.Snappy/compress ba)) (decompress [_ ba] (org.iq80.snappy.Snappy/uncompress ba 0 (alength ^bytes ba)))) @@ -29,30 +33,34 @@ (deftype LZMA2Compressor [compression-level] ;; Compression level ∈ℕ[0,9] (low->high) with 6 LZMA2 default (we use 0) ICompressor - (compress [_ ba] - (let [ba-len (alength ^bytes ba) - ba-os (ByteArrayOutputStream.) + (header-id [_] :lzma2) + (compress [_ ba] + (let [baos (ByteArrayOutputStream.) + dos (DataOutputStream. baos) + ;; + len-decomp (alength ^bytes ba) ;; Prefix with uncompressed length: - _ (.writeInt (DataOutputStream. ba-os) ba-len) - xzs (org.tukaani.xz.XZOutputStream. ba-os - (org.tukaani.xz.LZMA2Options. compression-level))] + _ (.writeInt dos len-decomp) + xzs (org.tukaani.xz.XZOutputStream. baos + (org.tukaani.xz.LZMA2Options. compression-level))] (.write xzs ^bytes ba) (.close xzs) - (.toByteArray ba-os))) + (.toByteArray baos))) (decompress [_ ba] - (let [ba-is (ByteArrayInputStream. ba) - ba-len (.readInt (DataInputStream. ba-is)) - ba (byte-array ba-len) - xzs (org.tukaani.xz.XZInputStream. ba-is)] - (.read xzs ba 0 ba-len) + (let [bais (ByteArrayInputStream. ba) + dis (DataInputStream. bais) + ;; + len-decomp (.readInt dis) + ba (byte-array len-decomp) + xzs (org.tukaani.xz.XZInputStream. bais)] + (.read xzs ba 0 len-decomp) (when (not= -1 (.read xzs)) ; Good practice as extra safety measure - (throw (Exception. "LZMA2 Decompress failed: corrupt data?"))) + (throw (ex-info "LZMA2 Decompress failed: corrupt data?" {:ba ba}))) ba))) (def lzma2-compressor - "Alpha - subject to change. - Default org.tukaani.xz.LZMA2 compressor: + "Default org.tukaani.xz.LZMA2 compressor: Ratio: high. Write speed: _very_ slow (also currently single-threaded). Read speed: slow. @@ -64,7 +72,8 @@ (deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor ^net.jpountz.lz4.LZ4Decompressor decompressor] ICompressor - (compress [_ ba] + (header-id [_] :lz4) + (compress [_ ba] (let [len-decomp (alength ^bytes ba) max-len-comp (.maxCompressedLength compressor len-decomp) ba-comp* (byte-array max-len-comp) ; Over-sized @@ -82,10 +91,10 @@ ;; len-decomp (.readInt dis) len-comp (- (alength ^bytes ba) 4) - ba-comp (byte-array len-comp) - _ (.readFully dis ba-comp 0 len-comp) + ;; ba-comp (byte-array len-comp) + ;; _ (.readFully dis ba-comp 0 len-comp) ba-decomp (byte-array len-decomp) - _ (.decompress decompressor ba-comp 0 ba-decomp 0 len-decomp)] + _ (.decompress decompressor ba 4 ba-decomp 0 len-decomp)] ba-decomp))) (def ^:private ^net.jpountz.lz4.LZ4Factory lz4-factory @@ -118,13 +127,13 @@ (count ba-bench)))}) (println - {:snappy (bench1 snappy-compressor) - ;; :lzma (bench1 lzma2-compressor) ; Slow! - :lz4 (bench1 lz4-compressor) - :lz4hc (bench1 lz4hc-compressor)}) + {:snappy (bench1 snappy-compressor) + ;:lzma2 (bench1 lzma2-compressor) ; Slow! + :lz4 (bench1 lz4-compressor) + :lz4hc (bench1 lz4hc-compressor)}) - ;;; 2014 April 5, initial benchmarks - {:snappy {:time 2214 :ratio 0.848} - :lzma {:time 46684 :ratio 0.494} - :lz4 {:time 1363 :ratio 0.819} - :lz4hc {:time 6045 :ratio 0.763}}) + ;;; 2014 April 7 + {:snappy {:time 2251, :ratio 0.852}, + :lzma2 {:time 46684 :ratio 0.494} + :lz4 {:time 1184, :ratio 0.819}, + :lz4hc {:time 5422, :ratio 0.761}}) diff --git a/src/taoensso/nippy/encryption.clj b/src/taoensso/nippy/encryption.clj index 5516783..cc715c4 100644 --- a/src/taoensso/nippy/encryption.clj +++ b/src/taoensso/nippy/encryption.clj @@ -1,13 +1,15 @@ (ns taoensso.nippy.encryption - "Alpha - subject to change. - Simple no-nonsense crypto with reasonable defaults. Because your Clojure data + "Simple no-nonsense crypto with reasonable defaults. Because your Clojure data deserves some privacy." {:author "Peter Taoussanis"} (:require [taoensso.encore :as encore])) ;;;; Interface +(def standard-header-ids "These'll support :auto thaw." #{:aes128-sha512}) + (defprotocol IEncryptor + (header-id [encryptor]) (encrypt ^bytes [encryptor pwd ba]) (decrypt ^bytes [encryptor pwd ba])) @@ -25,7 +27,7 @@ (defn- rand-bytes [size] (let [seed (byte-array size)] (.nextBytes prng seed) seed)) -;;;; Default keygen +;;;; Default key-gen (defn- sha512-key "SHA512-based key generator. Good JVM availability without extra dependencies @@ -38,6 +40,7 @@ (recur (.digest sha512-md ba) (dec n)) (-> ba (java.util.Arrays/copyOf aes128-block-size) (javax.crypto.spec.SecretKeySpec. "AES"))))) + (comment (time (sha512-key nil "hi" 1)) ; ~40ms per hash (fast) (time (sha512-key nil "hi" 5)) ; ~180ms (default) @@ -47,54 +50,52 @@ ;;;; Default implementations -(defn- destructure-typed-pwd - [typed-password] - (letfn [(throw-ex [] - (throw (AssertionError. - (str "Expected password form: " - "[<#{:salted :cached}> ].\n " - "See `default-aes128-encryptor` docstring for details!"))))] - (if-not (vector? typed-password) - (throw-ex) +(defn- destructure-typed-pwd [typed-password] + (let [throw-ex + (fn [] (throw (ex-info + (str "Expected password form: " + "[<#{:salted :cached}> ].\n " + "See `default-aes128-encryptor` docstring for details!") + {:typed-password typed-password})))] + (if-not (vector? typed-password) (throw-ex) (let [[type password] typed-password] - (if-not (#{:salted :cached} type) - (throw-ex) + (if-not (#{:salted :cached} type) (throw-ex) [type password]))))) (comment (destructure-typed-pwd [:salted "foo"])) -(defrecord AES128Encryptor [key-cache] +(defrecord AES128Encryptor [key-gen key-cache] IEncryptor - (encrypt [this typed-pwd data-ba] + (header-id [_] (if (= key-gen sha512-key) :aes128-sha512 :aes128-other)) + (encrypt [_ typed-pwd data-ba] (let [[type pwd] (destructure-typed-pwd typed-pwd) - salt? (= type :salted) + salt? (identical? type :salted) iv-ba (rand-bytes aes128-block-size) salt-ba (when salt? (rand-bytes salt-size)) prefix-ba (if-not salt? iv-ba (encore/ba-concat iv-ba salt-ba)) - key (encore/memoized (when-not salt? (:key-cache this)) - sha512-key salt-ba pwd) + key (encore/memoized (when-not salt? key-cache) + key-gen salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] (.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE ^javax.crypto.spec.SecretKeySpec key iv) (encore/ba-concat prefix-ba (.doFinal aes128-cipher data-ba)))) - (decrypt [this typed-pwd ba] + (decrypt [_ typed-pwd ba] (let [[type pwd] (destructure-typed-pwd typed-pwd) salt? (= type :salted) prefix-size (+ aes128-block-size (if salt? salt-size 0)) [prefix-ba data-ba] (encore/ba-split ba prefix-size) [iv-ba salt-ba] (if-not salt? [prefix-ba nil] (encore/ba-split prefix-ba aes128-block-size)) - key (encore/memoized (when-not salt? (:key-cache this)) - sha512-key salt-ba pwd) + key (encore/memoized (when-not salt? key-cache) + key-gen salt-ba pwd) iv (javax.crypto.spec.IvParameterSpec. iv-ba)] (.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE ^javax.crypto.spec.SecretKeySpec key iv) (.doFinal aes128-cipher data-ba)))) (def aes128-encryptor - "Alpha - subject to change. - Default 128bit AES encryptor with multi-round SHA-512 keygen. + "Default 128bit AES encryptor with multi-round SHA-512 key-gen. Password form [:salted \"my-password\"] --------------------------------------- @@ -128,7 +129,7 @@ Faster than `aes128-salted`, and harder to attack any particular key - but increased danger if a key is somehow compromised." - (->AES128Encryptor (atom {}))) + (->AES128Encryptor sha512-key (atom {}))) ;;;; Default implementation diff --git a/src/taoensso/nippy/tools.clj b/src/taoensso/nippy/tools.clj index a6f7516..ea1c84c 100644 --- a/src/taoensso/nippy/tools.clj +++ b/src/taoensso/nippy/tools.clj @@ -1,7 +1,6 @@ (ns taoensso.nippy.tools - "Alpha - subject to change. - Utilities for third-party tools that want to add fully-user-configurable Nippy - support. Used by Carmine and Faraday." + "Utilities for third-party tools that want to add fully-user-configurable + Nippy support. Used by Carmine and Faraday." {:author "Peter Taoussanis"} (:require [taoensso.nippy :as nippy])) @@ -23,26 +22,12 @@ (comment (freeze (wrap-for-freezing "wrapped")) (freeze "unwrapped")) -(defrecord EncryptedFrozen [ba]) -(defn encrypted-frozen? [x] (instance? EncryptedFrozen x)) - (def ^:dynamic *thaw-opts* nil) (defmacro with-thaw-opts "Evaluates body using given options for any automatic deserialization in context." [opts & body] `(binding [*thaw-opts* ~opts] ~@body)) -(defn thaw - "Like `nippy/thaw` but takes options from *thaw-opts* binding, and wraps - encrypted bytes for easy identification when no password has been provided - for decryption." +(defn thaw "Like `nippy/thaw` but takes options from *thaw-opts* binding." [ba & [{:keys [default-opts]}]] - (let [result (nippy/thaw ba (merge default-opts *thaw-opts* - {:taoensso.nippy/tools-thaw? true}))] - (if (= result :taoensso.nippy/need-password) - (EncryptedFrozen. ba) - result))) - -(comment (thaw (nippy/freeze "c" {:password [:cached "p"]})) - (with-thaw-opts {:password [:cached "p"]} - (thaw (nippy/freeze "c" {:password [:cached "p"]})))) + (nippy/thaw ba (merge default-opts *thaw-opts*))) diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 2a80fd4..561cd5e 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -4,7 +4,6 @@ [clojure.test.check.properties :as check-props] [expectations :as test :refer :all] [taoensso.nippy :as nippy :refer (freeze thaw)] - [taoensso.nippy.compression :as compression] [taoensso.nippy.benchmarks :as benchmarks])) (comment (test/run-tests '[taoensso.nippy.tests.main])) @@ -16,29 +15,30 @@ ;;;; Core (expect test-data ((comp thaw freeze) test-data)) -(expect test-data ((comp #(thaw % {:headerless-meta {:version 1 - :compressed? true - :encrypted? false}}) +(expect test-data ((comp #(thaw % {}) #(freeze % {:legacy-mode true})) test-data)) (expect test-data ((comp #(thaw % {:password [:salted "p"]}) #(freeze % {:password [:salted "p"]})) test-data)) -(expect test-data ((comp #(thaw % {:compressor compression/lzma2-compressor}) - #(freeze % {:compressor compression/lzma2-compressor})) +(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor}) + #(freeze % {:compressor nippy/lzma2-compressor})) test-data)) -(expect test-data ((comp #(thaw % {:compressor compression/lzma2-compressor +(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor :password [:salted "p"]}) - #(freeze % {:compressor compression/lzma2-compressor + #(freeze % {:compressor nippy/lzma2-compressor :password [:salted "p"]})) test-data)) +(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor}) + #(freeze % {:compressor nippy/lz4hc-compressor})) + test-data)) (expect ; Try roundtrip anything that simple-check can dream up (:result (check/quick-check 80 ; Time is n-non-linear (check-props/for-all [val check-gen/any] (= val (nippy/thaw (nippy/freeze val))))))) -(expect AssertionError (thaw (freeze test-data {:password "malformed"}))) +(expect Exception (thaw (freeze test-data {:password "malformed"}))) (expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) (expect Exception (thaw (freeze test-data {:password [:salted "p"]}) {:compressor nil}))