luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Experimental optimization: zero-copy freeze mode

Browse source
This commit is contained in:
Peter Taoussanis 2015-09-29 10:44:52 +07:00
parent fa17eb3a78
commit 89c9328596
1 changed files with 33 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
src/taoensso/nippy.clj
View file

@ -378,7 +378,6 @@
(defn- wrap-header [data-ba head-meta] (defn- wrap-header [data-ba head-meta]
(if-let [head-ba (get-head-ba head-meta)] (if-let [head-ba (get-head-ba head-meta)]
;; TODO Would be nice if we could avoid the array copy here:
(encore/ba-concat head-ba data-ba) (encore/ba-concat head-ba data-ba)
(throw (ex-info (format "Unrecognized header meta: %s" head-meta) (throw (ex-info (format "Unrecognized header meta: %s" head-meta)
{:head-meta head-meta})))) {:head-meta head-meta}))))
@ -425,32 +424,44 @@
compressor (if legacy-mode? snappy-compressor compressor) compressor (if legacy-mode? snappy-compressor compressor)
encryptor (when password (if-not legacy-mode? encryptor nil)) encryptor (when password (if-not legacy-mode? encryptor nil))
skip-header? (or skip-header? legacy-mode?) skip-header? (or skip-header? legacy-mode?)
zero-copy-mode? (and (nil? compressor) (nil? encryptor))
baos (ByteArrayOutputStream. 64) baos (ByteArrayOutputStream. 64)
dos (DataOutputStream. baos)] dos (DataOutputStream. baos)]
(freeze-to-out! dos x)
(let [ba (.toByteArray baos)
compressor (if zero-copy-mode?
(if (identical? compressor :auto) (do ; Optimized case
(if skip-header? (when-not skip-header? ; Avoid `wrap-header`'s array copy:
lz4-compressor (let [head-ba (get-head-ba {:compressor-id nil :encryptor-id nil})]
(*default-freeze-compressor-selector* ba)) (.write dos head-ba 0 4)))
(if (fn? compressor) (freeze-to-out! dos x)
(compressor ba) ; Assume compressor selector fn (.toByteArray baos))
compressor ; Assume compressor
))
ba (if compressor (compress compressor ba) ba) (do
ba (if encryptor (encrypt encryptor password ba) ba)] (freeze-to-out! dos x)
(let [ba (.toByteArray baos)
(if skip-header? ba compressor
(wrap-header ba (if (identical? compressor :auto)
{:compressor-id (when-let [c compressor] (if skip-header?
(or (compression/standard-header-ids lz4-compressor
(compression/header-id c)) :else)) (*default-freeze-compressor-selector* ba))
:encryptor-id (when-let [e encryptor] (if (fn? compressor)
(or (encryption/standard-header-ids (compressor ba) ; Assume compressor selector fn
(encryption/header-id e)) :else))})))))) compressor ; Assume compressor
))
ba (if compressor (compress compressor ba) ba)
ba (if encryptor (encrypt encryptor password ba) ba)]
(if skip-header?
ba
(wrap-header ba
{:compressor-id (when-let [c compressor]
(or (compression/standard-header-ids
(compression/header-id c)) :else))
:encryptor-id (when-let [e encryptor]
(or (encryption/standard-header-ids
(encryption/header-id e)) :else))}))))))))
;;;; Thawing ;;;; Thawing