luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LZ4: use fast decompressor

Browse source
This commit is contained in:
Peter Taoussanis 2014-04-05 16:18:51 +07:00
parent a6aba2c92a
commit 5b9358acf9
1 changed files with 30 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
src/taoensso/nippy/compression.clj
View file

@ -61,24 +61,32 @@
environments."
(->LZMA2Compressor 0))
(deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor
^net.jpountz.lz4.LZ4SafeDecompressor decompressor]
(deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor
^net.jpountz.lz4.LZ4Decompressor decompressor]
ICompressor
(compress [_ ba]
(let [in-len (alength ^bytes ba)
max-out-len (.maxCompressedLength compressor in-len)
ba-out* (byte-array max-out-len)
out-len (.compress compressor ba 0 in-len ba-out* 0 max-out-len)
ba-out (java.util.Arrays/copyOf ba-out* out-len)]
ba-out))
(let [len-decomp (alength ^bytes ba)
max-len-comp (.maxCompressedLength compressor len-decomp)
ba-comp* (byte-array max-len-comp) ; Over-sized
len-comp (.compress compressor ba 0 len-decomp ba-comp* 0 max-len-comp)
;;
baos (ByteArrayOutputStream. (+ len-comp 4))
dos (DataOutputStream. baos)]
(.writeInt dos len-decomp) ; Prefix with uncompressed length
(.write dos ba-comp* 0 len-comp)
(.toByteArray baos)))
(decompress [_ ba]
(let [in-len (alength ^bytes ba)
max-out-len in-len
ba-out* (byte-array (* max-out-len 3.0)) ; Nb over-sized!
out-len (.decompress decompressor ba 0 in-len ba-out* 0)
ba-out (java.util.Arrays/copyOf ba-out* out-len)]
ba-out)))
(let [bais (ByteArrayInputStream. ba)
dis (DataInputStream. bais)
;;
len-decomp (.readInt dis)
len-comp (- (alength ^bytes ba) 4)
ba-comp (byte-array len-comp)
_ (.readFully dis ba-comp 0 len-comp)
ba-decomp (byte-array len-decomp)
_ (.decompress decompressor ba-comp 0 ba-decomp 0 len-decomp)]
ba-decomp)))
(def ^:private ^net.jpountz.lz4.LZ4Factory lz4-factory
(net.jpountz.lz4.LZ4Factory/fastestInstance))
@ -89,13 +97,16 @@
Write speed: very high.
Read speed: very high.
A good general-purpose compressor, competitive with Snappy."
A good general-purpose compressor, competitive with Snappy.
Thanks to Max Penet (@mpenet) for our first implementation,
Ref. https://github.com/mpenet/nippy-lz4"
(->LZ4Compressor (.fastCompressor lz4-factory)
(.safeDecompressor lz4-factory)))
(.fastDecompressor lz4-factory)))
(def lz4hc-compressor "Like `lz4-compressor` but trades some speed for ratio."
(->LZ4Compressor (.highCompressor lz4-factory)
(.safeDecompressor lz4-factory)))
(.fastDecompressor lz4-factory)))
(comment
(def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8"))
@ -114,8 +125,5 @@
;;; 2014 April 5, initial benchmarks
{:snappy {:time 2214 :ratio 0.848}
:lzma {:time 46684 :ratio 0.494}
:lz4 {:time 1327 :ratio 0.819} ; w/o uncompressed size prefix
:lz4hc {:time 5762 :ratio 0.763} ; ''
;; :lz4 {:time 1404 :ratio 0.819} ; with uncompressed size prefix
;; :lz4hc {:time 6028 :ratio 0.763} ; ''
})
:lz4 {:time 1363 :ratio 0.819}
:lz4hc {:time 6045 :ratio 0.763}})