Encode compression type in Nippy header, major refactor/housekeeping
Housekeeping includes:
* Importing useful encryption+compression stuff into primary ns
for lib consumers.
* Promoting a number of things from Alpha status.
* Exceptions are now all `ex-info`s.
* Simplification of `thaw` API: Nippy v1 support is now automatic
& configuration-free (performance impact in most cases is negligible).
This commit is contained in:
Peter Taoussanis
parent
b7a454a9c8
commit
20b1c2b1d2
7 changed files with 275 additions and 335 deletions
|
|
@ -115,9 +115,9 @@ Couldn't be simpler!
|
|||
|
||||
See also the lower-level `freeze-to-out!` and `thaw-from-in!` fns for operating on `DataOutput` and `DataInput` types directly.
|
||||
|
||||
### Encryption (currently in **ALPHA**)
|
||||
### Encryption (v2+)
|
||||
|
||||
Nippy v2+ also gives you **dead simple data encryption**. Add a single option to your usual freeze/thaw calls like so:
|
||||
Nippy also gives you **dead simple data encryption**. Add a single option to your usual freeze/thaw calls like so:
|
||||
|
||||
```clojure
|
||||
(nippy/freeze nippy/stress-data {:password [:salted "my-password"]}) ; Encrypt
|
||||
|
|
@ -126,7 +126,7 @@ Nippy v2+ also gives you **dead simple data encryption**. Add a single option to
|
|||
|
||||
There's two default forms of encryption on offer: `:salted` and `:cached`. Each of these makes carefully-chosen trade-offs and is suited to one of two common use cases. See the `aes128-encryptor` [docstring](http://ptaoussanis.github.io/nippy/taoensso.nippy.encryption.html) for a detailed explanation of why/when you'd want one or the other.
|
||||
|
||||
### Custom types (v2.1+, ALPHA - subject to change)
|
||||
### Custom types (v2.1+)
|
||||
|
||||
```clojure
|
||||
(defrecord MyType [data])
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@
|
|||
[taoensso.encore :as encore]
|
||||
[taoensso.nippy
|
||||
(utils :as utils)
|
||||
(compression :as compression :refer (lz4-compressor
|
||||
snappy-compressor))
|
||||
(encryption :as encryption :refer (aes128-encryptor))])
|
||||
(compression :as compression)
|
||||
(encryption :as encryption)])
|
||||
(:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
|
||||
DataOutputStream Serializable ObjectOutputStream ObjectInputStream
|
||||
DataOutput DataInput]
|
||||
|
|
@ -21,25 +20,32 @@
|
|||
IRecord ISeq]))
|
||||
|
||||
;;;; Nippy 2.x+ header spec (4 bytes)
|
||||
;; Header is optional but recommended + enabled by default. Uses:
|
||||
;; Header is optional but recommended + enabled by default. Purpose:
|
||||
;; * Sanity check (data appears to be Nippy data).
|
||||
;; * Nippy version check (=> supports changes to data schema over time).
|
||||
;; * Encrypted &/or compressed data identification.
|
||||
;; * Supports :auto thaw compressor, encryptor.
|
||||
;;
|
||||
(def ^:private ^:const head-version 2)
|
||||
(def ^:private ^:const head-version 1)
|
||||
(def ^:private head-sig (.getBytes "NPY" "UTF-8"))
|
||||
(def ^:private ^:const head-meta "Final byte stores version-dependent metadata."
|
||||
{;;; Nippy <= v2.6 with Snappy as default compressor
|
||||
(byte 0) {:version 1 :compressed? false :encrypted? false}
|
||||
(byte 1) {:version 1 :compressed? true :encrypted? false}
|
||||
(byte 2) {:version 1 :compressed? false :encrypted? true}
|
||||
(byte 3) {:version 1 :compressed? true :encrypted? true}
|
||||
|
||||
;;; EXPERIMENTAL: Nippy >= v2.7 with LZ4 as default compressor
|
||||
(byte 4) {:version 2 :compressed? false :encrypted? false}
|
||||
(byte 5) {:version 2 :compressed? true :encrypted? false}
|
||||
(byte 6) {:version 2 :compressed? false :encrypted? true}
|
||||
(byte 7) {:version 2 :compressed? true :encrypted? true}})
|
||||
{(byte 0) {:version 1 :compressor-id nil :encryptor-id nil}
|
||||
(byte 4) {:version 1 :compressor-id nil :encryptor-id :else}
|
||||
(byte 5) {:version 1 :compressor-id :else :encryptor-id nil}
|
||||
(byte 6) {:version 1 :compressor-id :else :encryptor-id :else}
|
||||
;;
|
||||
(byte 2) {:version 1 :compressor-id nil :encryptor-id :aes128-sha512}
|
||||
;;
|
||||
(byte 1) {:version 1 :compressor-id :snappy :encryptor-id nil}
|
||||
(byte 3) {:version 1 :compressor-id :snappy :encryptor-id :aes128-sha512}
|
||||
(byte 7) {:version 1 :compressor-id :snappy :encryptor-id :else}
|
||||
;;
|
||||
(byte 8) {:version 1 :compressor-id :lz4 :encryptor-id nil}
|
||||
(byte 9) {:version 1 :compressor-id :lz4 :encryptor-id :aes128-sha512}
|
||||
(byte 10) {:version 1 :compressor-id :lz4 :encryptor-id :else}
|
||||
;;
|
||||
(byte 11) {:version 1 :compressor-id :lzma2 :encryptor-id nil}
|
||||
(byte 12) {:version 1 :compressor-id :lzma2 :encryptor-id :aes128-sha512}
|
||||
(byte 13) {:version 1 :compressor-id :lzma2 :encryptor-id :else}})
|
||||
|
||||
(defmacro when-debug-mode [& body] (when #_true false `(do ~@body)))
|
||||
|
||||
|
|
@ -86,7 +92,7 @@
|
|||
(def ^:const id-ratio (int 70))
|
||||
|
||||
(def ^:const id-record (int 80))
|
||||
;; (def ^:const id-type (int 81)) ; TODO
|
||||
;; (def ^:const id-type (int 81)) ; TODO?
|
||||
|
||||
(def ^:const id-date (int 90))
|
||||
(def ^:const id-uuid (int 91))
|
||||
|
|
@ -111,6 +117,21 @@
|
|||
(def ^:const id-old-keyword (int 12)) ; as of 2.0.0-alpha5, for str consistecy
|
||||
)
|
||||
|
||||
;;;; Ns imports (mostly for convenience of lib consumers)
|
||||
|
||||
(encore/defalias compress compression/compress)
|
||||
(encore/defalias decompress compression/decompress)
|
||||
(encore/defalias snappy-compressor compression/snappy-compressor)
|
||||
(encore/defalias lzma2-compressor compression/lzma2-compressor)
|
||||
(encore/defalias lz4-compressor compression/lz4-compressor)
|
||||
(encore/defalias lz4hc-compressor compression/lz4hc-compressor)
|
||||
|
||||
(encore/defalias encrypt encryption/encrypt)
|
||||
(encore/defalias decrypt encryption/decrypt)
|
||||
(encore/defalias aes128-encryptor encryption/aes128-encryptor)
|
||||
|
||||
(encore/defalias freezable? utils/freezable?)
|
||||
|
||||
;;;; Freezing
|
||||
|
||||
(defprotocol Freezable
|
||||
|
|
@ -137,7 +158,7 @@
|
|||
(let [x (with-meta x {:tag 'String})]
|
||||
`(write-bytes ~out (.getBytes ~x "UTF-8") ~small?)))
|
||||
|
||||
(defmacro write-compact-long "EXPERIMENTAL! Uses 2->9 bytes." [out x]
|
||||
(defmacro write-compact-long "Uses 2->9 bytes." [out x]
|
||||
`(write-bytes ~out (.toByteArray (java.math.BigInteger/valueOf (long ~x)))
|
||||
:small))
|
||||
|
||||
|
|
@ -304,19 +325,25 @@
|
|||
|
||||
:else ; Fallback #3: *final-freeze-fallback*
|
||||
(if-let [ffb *final-freeze-fallback*] (ffb x out)
|
||||
(throw (Exception. (format "Unfreezable type: %s %s"
|
||||
(type x) (str x))))))))
|
||||
(throw (ex-info (format "Unfreezable type: %s %s" (type x) (str x))
|
||||
{:type (type x)
|
||||
:as-str (pr-str x)}))))))
|
||||
|
||||
(def ^:private head-meta-id (reduce-kv #(assoc %1 %3 %2) {} head-meta))
|
||||
(def ^:private get-head-ba
|
||||
(memoize
|
||||
(fn [head-meta]
|
||||
(when-let [meta-id (get head-meta-id (assoc head-meta :version head-version))]
|
||||
(encore/ba-concat head-sig (byte-array [meta-id]))))))
|
||||
|
||||
(defn- wrap-header [data-ba metadata]
|
||||
(if-let [meta-id (head-meta-id (assoc metadata :version head-version))]
|
||||
(let [head-ba (encore/ba-concat head-sig (byte-array [meta-id]))]
|
||||
(encore/ba-concat head-ba data-ba))
|
||||
(throw (Exception. (str "Unrecognized header metadata: " metadata)))))
|
||||
(defn- wrap-header [data-ba head-meta]
|
||||
(if-let [head-ba (get-head-ba head-meta)]
|
||||
(encore/ba-concat head-ba data-ba)
|
||||
(throw (ex-info (format "Unrecognized header meta: %s" head-meta)
|
||||
{:head-meta head-meta}))))
|
||||
|
||||
(comment (wrap-header (.getBytes "foo") {:compressed? true
|
||||
:encrypted? false}))
|
||||
(comment (wrap-header (.getBytes "foo") {:compressor-id :lz4
|
||||
:encryptor-id nil}))
|
||||
|
||||
(defn freeze-to-out!
|
||||
"Low-level API. Serializes arg (any Clojure data type) to a DataOutput."
|
||||
|
|
@ -324,27 +351,30 @@
|
|||
(freeze-to-out data-output x))
|
||||
|
||||
(defn freeze
|
||||
"Serializes arg (any Clojure data type) to a byte array. For custom types
|
||||
extend the Clojure reader or see `extend-freeze`."
|
||||
^bytes [x & [{:keys [password compressor encryptor skip-header?]
|
||||
"Serializes arg (any Clojure data type) to a byte array. To freeze custom
|
||||
types, extend the Clojure reader or see `extend-freeze`."
|
||||
^bytes [x & [{:keys [compressor encryptor password skip-header?]
|
||||
:or {compressor lz4-compressor
|
||||
encryptor aes128-encryptor}
|
||||
:as opts}]]
|
||||
(let [;;; Legacy mode is deprecated
|
||||
compressor (if-not (:legacy-mode opts) compressor snappy-compressor)
|
||||
encryptor (if-not (:legacy-mode opts) encryptor nil)
|
||||
;;
|
||||
skip-header? (or skip-header? (:legacy-mode opts) ; Deprecated
|
||||
)
|
||||
bas (ByteArrayOutputStream.)
|
||||
sout (DataOutputStream. bas)]
|
||||
(freeze-to-out! sout x)
|
||||
(let [ba (.toByteArray bas)
|
||||
ba (if compressor (compression/compress compressor ba) ba)
|
||||
ba (if password (encryption/encrypt encryptor password ba) ba)]
|
||||
(let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze
|
||||
compressor (if-not legacy-mode? compressor snappy-compressor)
|
||||
encryptor (when password (if-not legacy-mode? encryptor nil))
|
||||
skip-header? (or skip-header? legacy-mode?)
|
||||
baos (ByteArrayOutputStream.)
|
||||
dos (DataOutputStream. baos)]
|
||||
(freeze-to-out! dos x)
|
||||
(let [ba (.toByteArray baos)
|
||||
ba (if-not compressor ba (compress compressor ba))
|
||||
ba (if-not encryptor ba (encrypt encryptor password ba))]
|
||||
(if skip-header? ba
|
||||
(wrap-header ba {:compressed? (boolean compressor)
|
||||
:encrypted? (boolean password)})))))
|
||||
(wrap-header ba
|
||||
{:compressor-id (when-let [c compressor]
|
||||
(or (compression/standard-header-ids
|
||||
(compression/header-id c)) :else))
|
||||
:encryptor-id (when-let [e encryptor]
|
||||
(or (encryption/standard-header-ids
|
||||
(encryption/header-id e)) :else))})))))
|
||||
|
||||
;;;; Thawing
|
||||
|
||||
|
|
@ -362,8 +392,7 @@
|
|||
(defmacro read-utf8 [in & [small?]]
|
||||
`(String. (read-bytes ~in ~small?) "UTF-8"))
|
||||
|
||||
(defmacro read-compact-long "EXPERIMENTAL!" [in]
|
||||
`(long (BigInteger. (read-bytes ~in :small))))
|
||||
(defmacro read-compact-long [in] `(long (BigInteger. (read-bytes ~in :small))))
|
||||
|
||||
(defmacro ^:private read-coll [in coll]
|
||||
`(let [in# ~in] (encore/repeatedly-into* ~coll (.readInt in#) (thaw-from-in in#))))
|
||||
|
|
@ -463,19 +492,25 @@
|
|||
id-old-keyword (keyword (.readUTF in))
|
||||
|
||||
(if-not (neg? type-id)
|
||||
(throw (Exception. (str "Unknown type ID: " type-id)))
|
||||
(throw (ex-info (format "Unknown type ID: %s" type-id)
|
||||
{:type-id type-id}))
|
||||
|
||||
;; Custom types
|
||||
(if-let [reader (get @custom-readers type-id)]
|
||||
(try (reader in)
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Reader exception for custom type ID: "
|
||||
(- type-id)) e))))
|
||||
(throw (Exception. (str "No reader provided for custom type ID: "
|
||||
(- type-id)))))))
|
||||
(throw (ex-info
|
||||
(format "Reader exception for custom type ID: %s"
|
||||
(- type-id))
|
||||
{:type-id (- type-id)} e))))
|
||||
(throw (ex-info
|
||||
(format "No reader provided for custom type ID: %s"
|
||||
(- type-id))
|
||||
{:type-id (- type-id)})))))
|
||||
|
||||
(catch Exception e
|
||||
(throw (Exception. (format "Thaw failed against type-id: %s" type-id) e))))))
|
||||
(throw (ex-info (format "Thaw failed against type-id: %s" type-id)
|
||||
{:type-id type-id} e))))))
|
||||
|
||||
(defn thaw-from-in!
|
||||
"Low-level API. Deserializes a frozen object from given DataInput to its
|
||||
|
|
@ -486,121 +521,118 @@
|
|||
(defn- try-parse-header [ba]
|
||||
(when-let [[head-ba data-ba] (encore/ba-split ba 4)]
|
||||
(let [[head-sig* [meta-id]] (encore/ba-split head-ba 3)]
|
||||
(when (encore/ba= head-sig* head-sig) ; Appears to be well-formed
|
||||
[data-ba (head-meta meta-id {:unrecognized-meta? true})]))))
|
||||
(when (encore/ba= head-sig* head-sig) ; Header appears to be well-formed
|
||||
[data-ba (get head-meta meta-id {:unrecognized-meta? true})]))))
|
||||
|
||||
(defn- get-auto-compressor [compressor-id]
|
||||
(case compressor-id
|
||||
nil nil
|
||||
:snappy snappy-compressor
|
||||
:lzma2 lzma2-compressor
|
||||
:lz4 lz4-compressor
|
||||
:no-header (throw (ex-info ":auto not supported on headerless data." {}))
|
||||
:else (throw (ex-info ":auto not supported for non-standard compressors." {}))
|
||||
(throw (ex-info (format "Unrecognized :auto compressor id: %s" compressor-id)
|
||||
{:compressor-id compressor-id}))))
|
||||
|
||||
(defn- get-auto-encryptor [encryptor-id]
|
||||
(case encryptor-id
|
||||
nil nil
|
||||
:aes128-sha512 aes128-encryptor
|
||||
:no-header (throw (ex-info ":auto not supported on headerless data." {}))
|
||||
:else (throw (ex-info ":auto not supported for non-standard encryptors."))
|
||||
(throw (ex-info (format "Unrecognized :auto encryptor id: %s" encryptor-id)
|
||||
{:encryptor-id encryptor-id}))))
|
||||
|
||||
(defn thaw
|
||||
"Deserializes a frozen object from given byte array to its original Clojure
|
||||
data type. By default[1] supports data frozen with current and all previous
|
||||
versions of Nippy. For custom types extend the Clojure reader or see
|
||||
`extend-thaw`.
|
||||
data type. Supports data frozen with current and all previous versions of
|
||||
Nippy. To thaw custom types, extend the Clojure reader or see `extend-thaw`.
|
||||
|
||||
[1] :headerless-meta provides a fallback facility for data frozen without a
|
||||
standard Nippy header (notably all Nippy v1 data). A default is provided for
|
||||
Nippy v1 thaw compatibility, but it's recommended that you _disable_ this
|
||||
fallback (`{:headerless-meta nil}`) if you're certain you won't be thawing
|
||||
headerless data."
|
||||
[^bytes ba & [{:keys [password compressor encryptor headerless-meta]
|
||||
Options include:
|
||||
:compressor - An ICompressor, :auto (requires Nippy header), or nil.
|
||||
:encryptor - An IEncryptor, :auto (requires Nippy header), or nil."
|
||||
[^bytes ba & [{:keys [compressor encryptor password]
|
||||
:or {compressor :auto
|
||||
encryptor :auto
|
||||
headerless-meta ; Recommend set to nil when possible
|
||||
{:version 2
|
||||
:compressed? true
|
||||
:encrypted? false}}
|
||||
encryptor :auto}
|
||||
:as opts}]]
|
||||
|
||||
(let [headerless-meta (merge headerless-meta (:legacy-opts opts)) ; Deprecated
|
||||
_ (assert (or (nil? headerless-meta)
|
||||
(head-meta-id headerless-meta))
|
||||
"Bad :headerless-meta (should be nil or a valid `head-meta` value)")
|
||||
(assert (not (contains? opts :headerless-meta))
|
||||
":headerless-meta `thaw` option removed as of Nippy v2.7.")
|
||||
|
||||
(let [ex (fn [msg & [e]] (throw (ex-info (format "Thaw failed: %s" msg)
|
||||
{:opts (merge opts
|
||||
{:compressor compressor
|
||||
:encryptor encryptor})}
|
||||
e)))
|
||||
thaw-data
|
||||
(fn [data-ba compressor-id encryptor-id]
|
||||
(let [compressor (if-not (identical? compressor :auto) compressor
|
||||
(get-auto-compressor compressor-id))
|
||||
encryptor (if-not (identical? encryptor :auto) encryptor
|
||||
(get-auto-encryptor encryptor-id))]
|
||||
|
||||
(when (and encryptor (not password))
|
||||
(ex "Password required for decryption."))
|
||||
|
||||
ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
|
||||
try-thaw-data
|
||||
(fn [data-ba {:keys [compressed? encrypted? version]
|
||||
:as _head-or-headerless-meta}]
|
||||
(let [encryptor (when (and password encrypted?)
|
||||
(if-not (identical? encryptor :auto) encryptor
|
||||
aes128-encryptor))
|
||||
compressor (when compressed?
|
||||
(if-not (identical? compressor :auto) compressor
|
||||
(case (int version)
|
||||
1 snappy-compressor
|
||||
2 lz4-compressor)))]
|
||||
(try
|
||||
(let [ba data-ba
|
||||
ba (if encryptor (encryption/decrypt encryptor password ba) ba)
|
||||
ba (if compressor (compression/decompress compressor ba) ba)
|
||||
sin (DataInputStream. (ByteArrayInputStream. ba))]
|
||||
(thaw-from-in! sin))
|
||||
ba (if-not encryptor ba (decrypt encryptor password ba))
|
||||
ba (if-not compressor ba (decompress compressor ba))
|
||||
dis (DataInputStream. (ByteArrayInputStream. ba))]
|
||||
(thaw-from-in! dis))
|
||||
|
||||
(catch Exception e
|
||||
(cond
|
||||
encryptor (if head-meta (ex "Wrong password/encryptor?" e)
|
||||
(ex "Unencrypted data?" e))
|
||||
compressor (if head-meta (ex "Encrypted data or wrong compressor?" e)
|
||||
(ex "Uncompressed data?" e))
|
||||
:else (if head-meta (ex "Corrupt data?" e)
|
||||
(ex "Data may be unfrozen, corrupt, compressed &/or encrypted.")))))))]
|
||||
(ex "Decryption/decompression failure, or data unfrozen/damaged.")))))
|
||||
|
||||
(if-let [[data-ba {:keys [unrecognized-meta? compressed? encrypted?]
|
||||
thaw-nippy-v1-data ; A little hackish, but necessary
|
||||
(fn [data-ba]
|
||||
(try (thaw-data data-ba :snappy nil)
|
||||
(catch Exception _
|
||||
(thaw-data data-ba nil nil))))]
|
||||
|
||||
(if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?]
|
||||
:as head-meta}] (try-parse-header ba)]
|
||||
|
||||
(cond ; A well-formed header _appears_ to be present
|
||||
(and (not headerless-meta) ; Cautious. It's unlikely but possible the
|
||||
; header sig match was a fluke and not an
|
||||
; indication of a real, well-formed header.
|
||||
; May really be headerless.
|
||||
unrecognized-meta?)
|
||||
(ex "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?")
|
||||
|
||||
;;; It's still possible below that the header match was a fluke, but it's
|
||||
;;; _very_ unlikely. Therefore _not_ going to incl.
|
||||
;;; `(not headerless-meta)` conditions below.
|
||||
|
||||
(and compressed? (not compressor))
|
||||
(ex "Compressed data? Try again with compressor.")
|
||||
(and encrypted? (not password))
|
||||
(if (::tools-thaw? opts) ::need-password
|
||||
(ex "Encrypted data? Try again with password."))
|
||||
:else (try (try-thaw-data data-ba head-meta)
|
||||
(catch Exception e
|
||||
(if headerless-meta
|
||||
(try (try-thaw-data ba headerless-meta)
|
||||
(catch Exception _
|
||||
(throw e)))
|
||||
(throw e)))))
|
||||
;; A well-formed header _appears_ to be present (it's possible though
|
||||
;; unlikely that this is a fluke and data is actually headerless):
|
||||
(try (thaw-data data-ba compressor-id encryptor-id)
|
||||
(catch Exception e
|
||||
(try (thaw-nippy-v1-data)
|
||||
(catch Exception _
|
||||
(if unrecognized-meta?
|
||||
(ex "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?"
|
||||
e)
|
||||
(throw e))))))
|
||||
|
||||
;; Well-formed header definitely not present
|
||||
(if headerless-meta
|
||||
(try-thaw-data ba headerless-meta)
|
||||
(ex "Data may be unfrozen, corrupt, compressed &/or encrypted.")))))
|
||||
(try (thaw-nippy-v1-data ba)
|
||||
(catch Exception _
|
||||
(thaw-data ba :no-header :no-header))))))
|
||||
|
||||
(comment (thaw (freeze "hello"))
|
||||
(thaw (freeze "hello" {:compressor nil}))
|
||||
(thaw (freeze "hello" {:password [:salted "p"]})) ; ex
|
||||
(thaw (freeze "hello" {:password [:salted "p"]})) ; ex: no pwd
|
||||
(thaw (freeze "hello") {:password [:salted "p"]}))
|
||||
|
||||
;;;; Custom types
|
||||
|
||||
(defmacro extend-freeze
|
||||
"Alpha - subject to change.
|
||||
Extends Nippy to support freezing of a custom type (ideally concrete) with
|
||||
"Extends Nippy to support freezing of a custom type (ideally concrete) with
|
||||
id ∈[1, 128]:
|
||||
(defrecord MyType [data])
|
||||
(extend-freeze MyType 1 [x data-output]
|
||||
(.writeUTF [data-output] (:data x)))"
|
||||
[type custom-type-id [x out] & body]
|
||||
(assert (and (>= custom-type-id 1) (<= custom-type-id 128)))
|
||||
`(extend-type ~type
|
||||
Freezable
|
||||
`(extend-type ~type Freezable
|
||||
(~'freeze-to-out* [~x ~(with-meta out {:tag 'java.io.DataOutput})]
|
||||
(write-id ~out ~(int (- custom-type-id)))
|
||||
~@body)))
|
||||
|
||||
(defonce custom-readers (atom {})) ; {<custom-type-id> (fn [data-input]) ...}
|
||||
(defmacro extend-thaw
|
||||
"Alpha - subject to change.
|
||||
Extends Nippy to support thawing of a custom type with id ∈[1, 128]:
|
||||
"Extends Nippy to support thawing of a custom type with id ∈[1, 128]:
|
||||
(extend-thaw 1 [data-input]
|
||||
(->MyType (.readUTF data-input)))"
|
||||
[custom-type-id [in] & body]
|
||||
|
|
@ -623,16 +655,14 @@
|
|||
compress? (> ba-len 1024)]
|
||||
(.writeBoolean out compress?)
|
||||
(if-not compress? (write-bytes out ba)
|
||||
(let [ba* (compression/compress compression/lzma2-compressor ba)]
|
||||
(let [ba* (compress lzma2-compressor ba)]
|
||||
(write-bytes out ba*)))))
|
||||
|
||||
(extend-thaw 128 [in]
|
||||
(let [compressed? (.readBoolean in)
|
||||
ba (read-bytes in)]
|
||||
(thaw ba {:compressor compression/lzma2-compressor
|
||||
:headerless-meta {:version 2
|
||||
:compressed? compressed?
|
||||
:encrypted? false}})))
|
||||
ba (read-bytes in)]
|
||||
(thaw ba {:compressor (when compressed? lzma2-compressor)
|
||||
:encryptor nil})))
|
||||
|
||||
(comment
|
||||
(->> (apply str (repeatedly 1000 rand))
|
||||
|
|
@ -719,8 +749,6 @@
|
|||
|
||||
;;;; Tools
|
||||
|
||||
(encore/defalias freezable? utils/freezable?)
|
||||
|
||||
(defn inspect-ba "Alpha - subject to change."
|
||||
[ba & [thaw-opts]]
|
||||
(if-not (encore/bytes? ba) :not-ba
|
||||
|
|
@ -734,15 +762,15 @@
|
|||
[data-ba nippy-header] (or (try-parse-header unwrapped-ba)
|
||||
[unwrapped-ba :no-header])]
|
||||
|
||||
{:known-wrapper known-wrapper
|
||||
:nippy2-header nippy-header ; Nippy v1.x didn't have a header
|
||||
:thawable? (try (thaw unwrapped-ba thaw-opts) true
|
||||
(catch Exception _ false))
|
||||
:unwrapped-ba unwrapped-ba
|
||||
:data-ba data-ba
|
||||
:unwrapped-size (alength ^bytes unwrapped-ba)
|
||||
:ba-size (alength ^bytes ba)
|
||||
:data-size (alength ^bytes data-ba)})))
|
||||
{:known-wrapper known-wrapper
|
||||
:nippy-v2-header nippy-header ; Nippy v1.x didn't have a header
|
||||
:thawable? (try (thaw unwrapped-ba thaw-opts) true
|
||||
(catch Exception _ false))
|
||||
:unwrapped-ba unwrapped-ba
|
||||
:data-ba data-ba
|
||||
:unwrapped-size (alength ^bytes unwrapped-ba)
|
||||
:ba-size (alength ^bytes ba)
|
||||
:data-size (alength ^bytes data-ba)})))
|
||||
|
||||
(comment (inspect-ba (freeze "hello"))
|
||||
(seq (:data-ba (inspect-ba (freeze "hello")))))
|
||||
|
|
@ -754,17 +782,3 @@
|
|||
|
||||
(def thaw-from-stream! "DEPRECATED: Use `thaw-from-in!` instead."
|
||||
thaw-from-in!)
|
||||
|
||||
(defn freeze-to-bytes "DEPRECATED: Use `freeze` instead."
|
||||
^bytes [x & {:keys [compress?]
|
||||
:or {compress? true}}]
|
||||
(freeze x {:skip-header? true
|
||||
:compressor (when compress? snappy-compressor)
|
||||
:password nil}))
|
||||
|
||||
(defn thaw-from-bytes "DEPRECATED: Use `thaw` instead."
|
||||
[ba & {:keys [compressed?]
|
||||
:or {compressed? true}}]
|
||||
(thaw ba {:headerless-opts {:compressed? compressed?}
|
||||
:compressor snappy-compressor
|
||||
:password nil}))
|
||||
|
|
|
|||
|
|
@ -3,8 +3,7 @@
|
|||
(:require [clojure.tools.reader.edn :as edn]
|
||||
[clojure.data.fressian :as fressian]
|
||||
[taoensso.encore :as encore]
|
||||
[taoensso.nippy :as nippy :refer (freeze thaw)]
|
||||
[taoensso.nippy.compression :as compression]))
|
||||
[taoensso.nippy :as nippy :refer (freeze thaw)]))
|
||||
|
||||
(def data nippy/stress-data-benchable)
|
||||
|
||||
|
|
@ -45,16 +44,14 @@
|
|||
#(thaw % {}))})
|
||||
(println {:fast (bench1 #(freeze % {:compressor nil
|
||||
:skip-header? true})
|
||||
#(thaw % {:headerless-meta
|
||||
{:version 2
|
||||
:compressed? false
|
||||
:encrypted? false}}))})
|
||||
#(thaw % {:compressor nil
|
||||
:encryptor nil}))})
|
||||
(println {:encrypted (bench1 #(freeze % {:password [:cached "p"]})
|
||||
#(thaw % {:password [:cached "p"]}))})
|
||||
|
||||
(when lzma2? ; Slow as molasses
|
||||
(println {:lzma2 (bench1 #(freeze % {:compressor compression/lzma2-compressor})
|
||||
#(thaw % {:compressor compression/lzma2-compressor}))}))
|
||||
(println {:lzma2 (bench1 #(freeze % {:compressor nippy/lzma2-compressor})
|
||||
#(thaw % {:compressor nippy/lzma2-compressor}))}))
|
||||
|
||||
(when fressian?
|
||||
(println {:fressian (bench1 fressian-freeze fressian-thaw)})))
|
||||
|
|
@ -70,87 +67,21 @@
|
|||
;; (bench {:laps 2})
|
||||
|
||||
;;; 2014 Apr 5 w/ headerless :fast, LZ4 replacing Snappy as default compressor
|
||||
;; {:default {:round 7669, :freeze 4157, :thaw 3512, :size 16143}}
|
||||
;; {:fast {:round 6918, :freeze 3636, :thaw 3282, :size 16992}}
|
||||
;; {:encrypted {:round 11814, :freeze 6180, :thaw 5634, :size 16164}}
|
||||
{:default {:round 7669, :freeze 4157, :thaw 3512, :size 16143}}
|
||||
{:fast {:round 6918, :freeze 3636, :thaw 3282, :size 16992}}
|
||||
{:encrypted {:round 11814, :freeze 6180, :thaw 5634, :size 16164}}
|
||||
|
||||
;;; 2014 Jan 22: with common-type size optimizations, enlarged stress-data
|
||||
;; {:reader {:round 109544, :freeze 39523, :thaw 70021, :size 27681}}
|
||||
;; {:default {:round 9234, :freeze 5128, :thaw 4106, :size 15989}}
|
||||
;; {:fast {:round 7402, :freeze 4021, :thaw 3381, :size 16957}}
|
||||
;; {:encrypted {:round 12594, :freeze 6884, :thaw 5710, :size 16020}}
|
||||
;; {:lzma2 {:round 66759, :freeze 44246, :thaw 22513, :size 11208}}
|
||||
;; {:fressian {:round 13052, :freeze 8694, :thaw 4358, :size 16942}}
|
||||
{:reader {:round 109544, :freeze 39523, :thaw 70021, :size 27681}}
|
||||
{:default {:round 9234, :freeze 5128, :thaw 4106, :size 15989}}
|
||||
{:fast {:round 7402, :freeze 4021, :thaw 3381, :size 16957}}
|
||||
{:encrypted {:round 12594, :freeze 6884, :thaw 5710, :size 16020}}
|
||||
{:lzma2 {:round 66759, :freeze 44246, :thaw 22513, :size 11208}}
|
||||
{:fressian {:round 13052, :freeze 8694, :thaw 4358, :size 16942}}
|
||||
|
||||
;;; 19 Oct 2013: Nippy v2.3.0, with lzma2 & (nb!) round=freeze+thaw
|
||||
;; {:reader {:round 67798, :freeze 23202, :thaw 44596, :size 22971}}
|
||||
;; {:default {:round 3632, :freeze 2349, :thaw 1283, :size 12369}}
|
||||
;; {:encrypted {:round 6970, :freeze 4073, :thaw 2897, :size 12388}}
|
||||
;; {:fast {:round 3294, :freeze 2109, :thaw 1185, :size 13277}}
|
||||
;; {:lzma2 {:round 44590, :freeze 29567, :thaw 15023, :size 9076}}
|
||||
|
||||
;;; 11 Oct 2013: Nippy v2.2.0, with both ztellman mods
|
||||
;; {:defaults {:round 4319, :freeze 2950, :thaw 1446, :data-size 12369}}
|
||||
;; {:encrypted {:round 7675, :freeze 4479, :thaw 3160, :data-size 12388}}
|
||||
;; {:fast {:round 3928, :freeze 2530, :thaw 1269, :data-size 13277}}
|
||||
;; {:defaults-delta {:round 0.84 :freeze 0.79 :thaw 1.14}} ; vs 2.2.0
|
||||
|
||||
;;; 11 Oct 2013: Nippy v2.2.0, with first ztellman mod
|
||||
;; {:defaults {:round 4059, :freeze 2578, :thaw 1351, :data-size 12342}}
|
||||
;; {:encrypted {:round 7248, :freeze 4058, :thaw 3041, :data-size 12372}}
|
||||
;; {:fast {:round 3430, :freeze 2085, :thaw 1229, :data-size 13277}}
|
||||
;; {:defaults-delta {:round 0.79 :freeze 0.69 :thaw 1.07}} ; vs 2.2.0
|
||||
|
||||
;;; 11 Oct 2013: Nippy v2.2.0
|
||||
;; {:defaults {:round 5135, :freeze 3711, :thaw 1266, :data-size 12393}}
|
||||
;; {:encrypted {:round 8655, :freeze 5323, :thaw 3036, :data-size 12420}}
|
||||
;; {:fast {:round 4670, :freeze 3282, :thaw 1294, :data-size 13277}}
|
||||
|
||||
;;; 7 Auguest 2013: Nippy v2.2.0-RC1
|
||||
;; {:reader {:round 71582, :freeze 13656, :thaw 56730, :data-size 22964}}
|
||||
;; {:defaults {:round 5619, :freeze 3710, :thaw 1783, :data-size 12368}}
|
||||
;; {:encrypted {:round 9113, :freeze 5324, :thaw 3500, :data-size 12388}}
|
||||
;; {:fast {:round 5130, :freeze 3286, :thaw 1667, :data-size 13325}}
|
||||
|
||||
;;; 17 June 2013: Clojure 1.5.1, JVM 7 Nippy 2.0.0-alpha6 w/fast io-streams
|
||||
;; {:reader {:round 49819, :freeze 23601, :thaw 26247, :data-size 22966}}
|
||||
;; {:defaults {:round 5670, :freeze 3536, :thaw 1919, :data-size 12396}}
|
||||
;; {:encrypted {:round 9038, :freeze 5111, :thaw 3582, :data-size 12420}}
|
||||
;; {:fast {:round 5182, :freeze 3177, :thaw 1820, :data-size 13342}}
|
||||
|
||||
;;; 16 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha6
|
||||
;; {:reader {:freeze 23601, :thaw 26247, :round 49819, :data-size 22966}}
|
||||
;; {:defaults {:freeze 3554, :thaw 2002, :round 5831, :data-size 12394}}
|
||||
;; {:encrypted {:freeze 5117, :thaw 3600, :round 9006, :data-size 12420}}
|
||||
;; {:fast {:freeze 3247, :thaw 1914, :round 5329, :data-size 13342}}
|
||||
|
||||
;;; 13 June 2013: Clojure 1.5.1, Nippy 2.0.0-alpha1
|
||||
;; {:reader {:freeze 23124, :thaw 26469, :round 47674, :data-size 22923}}
|
||||
;; {:defaults {:freeze 4007, :thaw 2520, :round 6038, :data-size 12387}}
|
||||
;; {:encrypted {:freeze 5560, :thaw 3867, :round 9157, :data-size 12405}}
|
||||
;; {:fast {:freeze 3429, :thaw 2078, :round 5577, :data-size 13237}}
|
||||
|
||||
;;; 11 June 2013: Clojure 1.5.1, Nippy 1.3.0-alpha1
|
||||
;; {:reader {:freeze 17042, :thaw 31579, :round 48379, :data-size 22954}}
|
||||
;; {:fast {:freeze 3078, :thaw 4684, :round 8117, :data-size 13274}}
|
||||
;; {:defaults {:freeze 3810, :thaw 5295, :round 9052, :data-size 12394}}
|
||||
;; {:encrypted {:freeze 5800, :thaw 6862, :round 12317, :data-size 12416}}
|
||||
|
||||
;;; Clojure 1.5.1, Nippy 1.2.1 (+ sorted-set, sorted-map)
|
||||
;; (def data (dissoc data :sorted-set :sorted-map))
|
||||
;; {:reader {:freeze 15037, :thaw 27885, :round 43945},
|
||||
;; :nippy {:freeze 3194, :thaw 4734, :round 8380}}
|
||||
;; {:reader-size 22975, :defaults-size 12400, :encrypted-size 12400}
|
||||
|
||||
;;; Clojure 1.4.0, Nippy 1.0.0 (+ tagged-uuid, tagged-date)
|
||||
;; {:reader {:freeze 22595, :thaw 31148, :round 54059}
|
||||
;; :nippy {:freeze 3324, :thaw 3725, :round 6918}}
|
||||
|
||||
;;; Clojure 1.3.0, Nippy 0.9.2
|
||||
;; {:reader {:freeze 28505, :thaw 36451, :round 59545},
|
||||
;; :nippy {:freeze 3751, :thaw 4184, :round 7769}}
|
||||
|
||||
(println (bench* (roundtrip data))) ; Snappy implementations
|
||||
;; {:no-snappy [6163 6064 6042 6176] :JNI [6489 6446 6542 6412]
|
||||
;; :native-array-copy [6569 6419 6414 6590]}
|
||||
)
|
||||
{:reader {:round 67798, :freeze 23202, :thaw 44596, :size 22971}}
|
||||
{:default {:round 3632, :freeze 2349, :thaw 1283, :size 12369}}
|
||||
{:encrypted {:round 6970, :freeze 4073, :thaw 2897, :size 12388}}
|
||||
{:fast {:round 3294, :freeze 2109, :thaw 1185, :size 13277}}
|
||||
{:lzma2 {:round 44590, :freeze 29567, :thaw 15023, :size 9076}})
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
(ns taoensso.nippy.compression "Alpha - subject to change."
|
||||
(ns taoensso.nippy.compression
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.encore :as encore])
|
||||
(:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
|
||||
|
|
@ -7,13 +7,17 @@
|
|||
;;;; Interface
|
||||
|
||||
(defprotocol ICompressor
|
||||
(header-id [compressor])
|
||||
(compress ^bytes [compressor ba])
|
||||
(decompress ^bytes [compressor ba]))
|
||||
|
||||
;;;; Default implementations
|
||||
|
||||
(def standard-header-ids "These'll support :auto thaw." #{:snappy :lzma2 :lz4})
|
||||
|
||||
(deftype SnappyCompressor []
|
||||
ICompressor
|
||||
(header-id [_] :snappy)
|
||||
(compress [_ ba] (org.iq80.snappy.Snappy/compress ba))
|
||||
(decompress [_ ba] (org.iq80.snappy.Snappy/uncompress ba 0 (alength ^bytes ba))))
|
||||
|
||||
|
|
@ -29,30 +33,34 @@
|
|||
(deftype LZMA2Compressor [compression-level]
|
||||
;; Compression level ∈ℕ[0,9] (low->high) with 6 LZMA2 default (we use 0)
|
||||
ICompressor
|
||||
(compress [_ ba]
|
||||
(let [ba-len (alength ^bytes ba)
|
||||
ba-os (ByteArrayOutputStream.)
|
||||
(header-id [_] :lzma2)
|
||||
(compress [_ ba]
|
||||
(let [baos (ByteArrayOutputStream.)
|
||||
dos (DataOutputStream. baos)
|
||||
;;
|
||||
len-decomp (alength ^bytes ba)
|
||||
;; Prefix with uncompressed length:
|
||||
_ (.writeInt (DataOutputStream. ba-os) ba-len)
|
||||
xzs (org.tukaani.xz.XZOutputStream. ba-os
|
||||
(org.tukaani.xz.LZMA2Options. compression-level))]
|
||||
_ (.writeInt dos len-decomp)
|
||||
xzs (org.tukaani.xz.XZOutputStream. baos
|
||||
(org.tukaani.xz.LZMA2Options. compression-level))]
|
||||
(.write xzs ^bytes ba)
|
||||
(.close xzs)
|
||||
(.toByteArray ba-os)))
|
||||
(.toByteArray baos)))
|
||||
|
||||
(decompress [_ ba]
|
||||
(let [ba-is (ByteArrayInputStream. ba)
|
||||
ba-len (.readInt (DataInputStream. ba-is))
|
||||
ba (byte-array ba-len)
|
||||
xzs (org.tukaani.xz.XZInputStream. ba-is)]
|
||||
(.read xzs ba 0 ba-len)
|
||||
(let [bais (ByteArrayInputStream. ba)
|
||||
dis (DataInputStream. bais)
|
||||
;;
|
||||
len-decomp (.readInt dis)
|
||||
ba (byte-array len-decomp)
|
||||
xzs (org.tukaani.xz.XZInputStream. bais)]
|
||||
(.read xzs ba 0 len-decomp)
|
||||
(when (not= -1 (.read xzs)) ; Good practice as extra safety measure
|
||||
(throw (Exception. "LZMA2 Decompress failed: corrupt data?")))
|
||||
(throw (ex-info "LZMA2 Decompress failed: corrupt data?" {:ba ba})))
|
||||
ba)))
|
||||
|
||||
(def lzma2-compressor
|
||||
"Alpha - subject to change.
|
||||
Default org.tukaani.xz.LZMA2 compressor:
|
||||
"Default org.tukaani.xz.LZMA2 compressor:
|
||||
Ratio: high.
|
||||
Write speed: _very_ slow (also currently single-threaded).
|
||||
Read speed: slow.
|
||||
|
|
@ -64,7 +72,8 @@
|
|||
(deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor
|
||||
^net.jpountz.lz4.LZ4Decompressor decompressor]
|
||||
ICompressor
|
||||
(compress [_ ba]
|
||||
(header-id [_] :lz4)
|
||||
(compress [_ ba]
|
||||
(let [len-decomp (alength ^bytes ba)
|
||||
max-len-comp (.maxCompressedLength compressor len-decomp)
|
||||
ba-comp* (byte-array max-len-comp) ; Over-sized
|
||||
|
|
@ -82,10 +91,10 @@
|
|||
;;
|
||||
len-decomp (.readInt dis)
|
||||
len-comp (- (alength ^bytes ba) 4)
|
||||
ba-comp (byte-array len-comp)
|
||||
_ (.readFully dis ba-comp 0 len-comp)
|
||||
;; ba-comp (byte-array len-comp)
|
||||
;; _ (.readFully dis ba-comp 0 len-comp)
|
||||
ba-decomp (byte-array len-decomp)
|
||||
_ (.decompress decompressor ba-comp 0 ba-decomp 0 len-decomp)]
|
||||
_ (.decompress decompressor ba 4 ba-decomp 0 len-decomp)]
|
||||
ba-decomp)))
|
||||
|
||||
(def ^:private ^net.jpountz.lz4.LZ4Factory lz4-factory
|
||||
|
|
@ -118,13 +127,13 @@
|
|||
(count ba-bench)))})
|
||||
|
||||
(println
|
||||
{:snappy (bench1 snappy-compressor)
|
||||
;; :lzma (bench1 lzma2-compressor) ; Slow!
|
||||
:lz4 (bench1 lz4-compressor)
|
||||
:lz4hc (bench1 lz4hc-compressor)})
|
||||
{:snappy (bench1 snappy-compressor)
|
||||
;:lzma2 (bench1 lzma2-compressor) ; Slow!
|
||||
:lz4 (bench1 lz4-compressor)
|
||||
:lz4hc (bench1 lz4hc-compressor)})
|
||||
|
||||
;;; 2014 April 5, initial benchmarks
|
||||
{:snappy {:time 2214 :ratio 0.848}
|
||||
:lzma {:time 46684 :ratio 0.494}
|
||||
:lz4 {:time 1363 :ratio 0.819}
|
||||
:lz4hc {:time 6045 :ratio 0.763}})
|
||||
;;; 2014 April 7
|
||||
{:snappy {:time 2251, :ratio 0.852},
|
||||
:lzma2 {:time 46684 :ratio 0.494}
|
||||
:lz4 {:time 1184, :ratio 0.819},
|
||||
:lz4hc {:time 5422, :ratio 0.761}})
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
(ns taoensso.nippy.encryption
|
||||
"Alpha - subject to change.
|
||||
Simple no-nonsense crypto with reasonable defaults. Because your Clojure data
|
||||
"Simple no-nonsense crypto with reasonable defaults. Because your Clojure data
|
||||
deserves some privacy."
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.encore :as encore]))
|
||||
|
||||
;;;; Interface
|
||||
|
||||
(def standard-header-ids "These'll support :auto thaw." #{:aes128-sha512})
|
||||
|
||||
(defprotocol IEncryptor
|
||||
(header-id [encryptor])
|
||||
(encrypt ^bytes [encryptor pwd ba])
|
||||
(decrypt ^bytes [encryptor pwd ba]))
|
||||
|
||||
|
|
@ -25,7 +27,7 @@
|
|||
|
||||
(defn- rand-bytes [size] (let [seed (byte-array size)] (.nextBytes prng seed) seed))
|
||||
|
||||
;;;; Default keygen
|
||||
;;;; Default key-gen
|
||||
|
||||
(defn- sha512-key
|
||||
"SHA512-based key generator. Good JVM availability without extra dependencies
|
||||
|
|
@ -38,6 +40,7 @@
|
|||
(recur (.digest sha512-md ba) (dec n))
|
||||
(-> ba (java.util.Arrays/copyOf aes128-block-size)
|
||||
(javax.crypto.spec.SecretKeySpec. "AES")))))
|
||||
|
||||
(comment
|
||||
(time (sha512-key nil "hi" 1)) ; ~40ms per hash (fast)
|
||||
(time (sha512-key nil "hi" 5)) ; ~180ms (default)
|
||||
|
|
@ -47,54 +50,52 @@
|
|||
|
||||
;;;; Default implementations
|
||||
|
||||
(defn- destructure-typed-pwd
|
||||
[typed-password]
|
||||
(letfn [(throw-ex []
|
||||
(throw (AssertionError.
|
||||
(str "Expected password form: "
|
||||
"[<#{:salted :cached}> <password-string>].\n "
|
||||
"See `default-aes128-encryptor` docstring for details!"))))]
|
||||
(if-not (vector? typed-password)
|
||||
(throw-ex)
|
||||
(defn- destructure-typed-pwd [typed-password]
|
||||
(let [throw-ex
|
||||
(fn [] (throw (ex-info
|
||||
(str "Expected password form: "
|
||||
"[<#{:salted :cached}> <password-string>].\n "
|
||||
"See `default-aes128-encryptor` docstring for details!")
|
||||
{:typed-password typed-password})))]
|
||||
(if-not (vector? typed-password) (throw-ex)
|
||||
(let [[type password] typed-password]
|
||||
(if-not (#{:salted :cached} type)
|
||||
(throw-ex)
|
||||
(if-not (#{:salted :cached} type) (throw-ex)
|
||||
[type password])))))
|
||||
|
||||
(comment (destructure-typed-pwd [:salted "foo"]))
|
||||
|
||||
(defrecord AES128Encryptor [key-cache]
|
||||
(defrecord AES128Encryptor [key-gen key-cache]
|
||||
IEncryptor
|
||||
(encrypt [this typed-pwd data-ba]
|
||||
(header-id [_] (if (= key-gen sha512-key) :aes128-sha512 :aes128-other))
|
||||
(encrypt [_ typed-pwd data-ba]
|
||||
(let [[type pwd] (destructure-typed-pwd typed-pwd)
|
||||
salt? (= type :salted)
|
||||
salt? (identical? type :salted)
|
||||
iv-ba (rand-bytes aes128-block-size)
|
||||
salt-ba (when salt? (rand-bytes salt-size))
|
||||
prefix-ba (if-not salt? iv-ba (encore/ba-concat iv-ba salt-ba))
|
||||
key (encore/memoized (when-not salt? (:key-cache this))
|
||||
sha512-key salt-ba pwd)
|
||||
key (encore/memoized (when-not salt? key-cache)
|
||||
key-gen salt-ba pwd)
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
|
||||
(.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE
|
||||
^javax.crypto.spec.SecretKeySpec key iv)
|
||||
(encore/ba-concat prefix-ba (.doFinal aes128-cipher data-ba))))
|
||||
|
||||
(decrypt [this typed-pwd ba]
|
||||
(decrypt [_ typed-pwd ba]
|
||||
(let [[type pwd] (destructure-typed-pwd typed-pwd)
|
||||
salt? (= type :salted)
|
||||
prefix-size (+ aes128-block-size (if salt? salt-size 0))
|
||||
[prefix-ba data-ba] (encore/ba-split ba prefix-size)
|
||||
[iv-ba salt-ba] (if-not salt? [prefix-ba nil]
|
||||
(encore/ba-split prefix-ba aes128-block-size))
|
||||
key (encore/memoized (when-not salt? (:key-cache this))
|
||||
sha512-key salt-ba pwd)
|
||||
key (encore/memoized (when-not salt? key-cache)
|
||||
key-gen salt-ba pwd)
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
|
||||
(.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE
|
||||
^javax.crypto.spec.SecretKeySpec key iv)
|
||||
(.doFinal aes128-cipher data-ba))))
|
||||
|
||||
(def aes128-encryptor
|
||||
"Alpha - subject to change.
|
||||
Default 128bit AES encryptor with multi-round SHA-512 keygen.
|
||||
"Default 128bit AES encryptor with multi-round SHA-512 key-gen.
|
||||
|
||||
Password form [:salted \"my-password\"]
|
||||
---------------------------------------
|
||||
|
|
@ -128,7 +129,7 @@
|
|||
|
||||
Faster than `aes128-salted`, and harder to attack any particular key - but
|
||||
increased danger if a key is somehow compromised."
|
||||
(->AES128Encryptor (atom {})))
|
||||
(->AES128Encryptor sha512-key (atom {})))
|
||||
|
||||
;;;; Default implementation
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
(ns taoensso.nippy.tools
|
||||
"Alpha - subject to change.
|
||||
Utilities for third-party tools that want to add fully-user-configurable Nippy
|
||||
support. Used by Carmine and Faraday."
|
||||
"Utilities for third-party tools that want to add fully-user-configurable
|
||||
Nippy support. Used by Carmine and Faraday."
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.nippy :as nippy]))
|
||||
|
||||
|
|
@ -23,26 +22,12 @@
|
|||
(comment (freeze (wrap-for-freezing "wrapped"))
|
||||
(freeze "unwrapped"))
|
||||
|
||||
(defrecord EncryptedFrozen [ba])
|
||||
(defn encrypted-frozen? [x] (instance? EncryptedFrozen x))
|
||||
|
||||
(def ^:dynamic *thaw-opts* nil)
|
||||
(defmacro with-thaw-opts
|
||||
"Evaluates body using given options for any automatic deserialization in
|
||||
context."
|
||||
[opts & body] `(binding [*thaw-opts* ~opts] ~@body))
|
||||
|
||||
(defn thaw
|
||||
"Like `nippy/thaw` but takes options from *thaw-opts* binding, and wraps
|
||||
encrypted bytes for easy identification when no password has been provided
|
||||
for decryption."
|
||||
(defn thaw "Like `nippy/thaw` but takes options from *thaw-opts* binding."
|
||||
[ba & [{:keys [default-opts]}]]
|
||||
(let [result (nippy/thaw ba (merge default-opts *thaw-opts*
|
||||
{:taoensso.nippy/tools-thaw? true}))]
|
||||
(if (= result :taoensso.nippy/need-password)
|
||||
(EncryptedFrozen. ba)
|
||||
result)))
|
||||
|
||||
(comment (thaw (nippy/freeze "c" {:password [:cached "p"]}))
|
||||
(with-thaw-opts {:password [:cached "p"]}
|
||||
(thaw (nippy/freeze "c" {:password [:cached "p"]}))))
|
||||
(nippy/thaw ba (merge default-opts *thaw-opts*)))
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
[clojure.test.check.properties :as check-props]
|
||||
[expectations :as test :refer :all]
|
||||
[taoensso.nippy :as nippy :refer (freeze thaw)]
|
||||
[taoensso.nippy.compression :as compression]
|
||||
[taoensso.nippy.benchmarks :as benchmarks]))
|
||||
|
||||
(comment (test/run-tests '[taoensso.nippy.tests.main]))
|
||||
|
|
@ -16,29 +15,30 @@
|
|||
;;;; Core
|
||||
|
||||
(expect test-data ((comp thaw freeze) test-data))
|
||||
(expect test-data ((comp #(thaw % {:headerless-meta {:version 1
|
||||
:compressed? true
|
||||
:encrypted? false}})
|
||||
(expect test-data ((comp #(thaw % {})
|
||||
#(freeze % {:legacy-mode true}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:password [:salted "p"]})
|
||||
#(freeze % {:password [:salted "p"]}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:compressor compression/lzma2-compressor})
|
||||
#(freeze % {:compressor compression/lzma2-compressor}))
|
||||
(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor})
|
||||
#(freeze % {:compressor nippy/lzma2-compressor}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:compressor compression/lzma2-compressor
|
||||
(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor
|
||||
:password [:salted "p"]})
|
||||
#(freeze % {:compressor compression/lzma2-compressor
|
||||
#(freeze % {:compressor nippy/lzma2-compressor
|
||||
:password [:salted "p"]}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor})
|
||||
#(freeze % {:compressor nippy/lz4hc-compressor}))
|
||||
test-data))
|
||||
|
||||
(expect ; Try roundtrip anything that simple-check can dream up
|
||||
(:result (check/quick-check 80 ; Time is n-non-linear
|
||||
(check-props/for-all [val check-gen/any]
|
||||
(= val (nippy/thaw (nippy/freeze val)))))))
|
||||
|
||||
(expect AssertionError (thaw (freeze test-data {:password "malformed"})))
|
||||
(expect Exception (thaw (freeze test-data {:password "malformed"})))
|
||||
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
|
||||
(expect Exception (thaw (freeze test-data {:password [:salted "p"]})
|
||||
{:compressor nil}))
|
||||
|
|
|
|||
Loading…
Reference in a new issue