luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Experimental optimization: allow smart, automatic compressor selection

Browse source
This commit is contained in:
Peter Taoussanis 2015-04-17 18:47:03 +07:00
parent a8a6c933b3
commit 0e691cbbcf
1 changed files with 26 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
src/taoensso/nippy.clj
View file

@ -369,11 +369,24 @@
[^DataOutput data-output x & _] [^DataOutput data-output x & _]
(freeze-to-out data-output x)) (freeze-to-out data-output x))
(defn default-freeze-compression-selector [^bytes ba]
(let [ba-len (alength ba)]
(cond
;; (> ba-len 1024) lzma2-compressor
;; (> ba-len 512) lz4hc-compressor
(> ba-len 128) lz4-compressor
:else nil)))
;; Allows easy global override of default (e.g. to enable lzma2 choices, etc.):
(encore/defonce* default-freeze-compression-selector_
"EXPERIMENTAL, undocumented."
(atom default-freeze-compression-selector))
(defn freeze (defn freeze
"Serializes arg (any Clojure data type) to a byte array. To freeze custom "Serializes arg (any Clojure data type) to a byte array. To freeze custom
types, extend the Clojure reader or see `extend-freeze`." types, extend the Clojure reader or see `extend-freeze`."
^bytes [x & [{:keys [compressor encryptor password skip-header?] ^bytes [x & [{:keys [compressor encryptor password skip-header?]
:or {compressor lz4-compressor :or {compressor :auto ; or (fn [^bytes ba]) -> compressor
encryptor aes128-encryptor} encryptor aes128-encryptor}
:as opts}]] :as opts}]]
(let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze (let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze
@ -384,8 +397,19 @@
dos (DataOutputStream. baos)] dos (DataOutputStream. baos)]
(freeze-to-out! dos x) (freeze-to-out! dos x)
(let [ba (.toByteArray baos) (let [ba (.toByteArray baos)
compressor
(if (identical? compressor :auto)
(if skip-header?
lz4-compressor
(@default-freeze-compression-selector_ ba))
(if (fn? compressor)
(compressor ba)
compressor))
ba (if-not compressor ba (compress compressor ba)) ba (if-not compressor ba (compress compressor ba))
ba (if-not encryptor ba (encrypt encryptor password ba))] ba (if-not encryptor ba (encrypt encryptor password ba))]
(if skip-header? ba (if skip-header? ba
(wrap-header ba (wrap-header ba
{:compressor-id (when-let [c compressor] {:compressor-id (when-let [c compressor]
@ -722,6 +746,7 @@
;;; Some useful custom types - EXPERIMENTAL ;;; Some useful custom types - EXPERIMENTAL
;; DEPRECATED in favour of smart compressor selection
(defrecord Compressable-LZMA2 [value]) (defrecord Compressable-LZMA2 [value])
(extend-freeze Compressable-LZMA2 128 [x out] (extend-freeze Compressable-LZMA2 128 [x out]
(let [ba (freeze (:value x) {:skip-header? true :compressor nil}) (let [ba (freeze (:value x) {:skip-header? true :compressor nil})