luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LZ4: use fast decompressor

Browse source
This commit is contained in:
Peter Taoussanis 2014-04-05 16:18:51 +07:00
parent a6aba2c92a
commit 5b9358acf9
1 changed files with 30 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
src/taoensso/nippy/compression.clj
View file

@ -61,24 +61,32 @@
environments." environments."
(->LZMA2Compressor 0)) (->LZMA2Compressor 0))
(deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor (deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor
^net.jpountz.lz4.LZ4SafeDecompressor decompressor] ^net.jpountz.lz4.LZ4Decompressor decompressor]
ICompressor ICompressor
(compress [_ ba] (compress [_ ba]
(let [in-len (alength ^bytes ba) (let [len-decomp (alength ^bytes ba)
max-out-len (.maxCompressedLength compressor in-len) max-len-comp (.maxCompressedLength compressor len-decomp)
ba-out* (byte-array max-out-len) ba-comp* (byte-array max-len-comp) ; Over-sized
out-len (.compress compressor ba 0 in-len ba-out* 0 max-out-len) len-comp (.compress compressor ba 0 len-decomp ba-comp* 0 max-len-comp)
ba-out (java.util.Arrays/copyOf ba-out* out-len)] ;;
ba-out)) baos (ByteArrayOutputStream. (+ len-comp 4))
dos (DataOutputStream. baos)]
(.writeInt dos len-decomp) ; Prefix with uncompressed length
(.write dos ba-comp* 0 len-comp)
(.toByteArray baos)))
(decompress [_ ba] (decompress [_ ba]
(let [in-len (alength ^bytes ba) (let [bais (ByteArrayInputStream. ba)
max-out-len in-len dis (DataInputStream. bais)
ba-out* (byte-array (* max-out-len 3.0)) ; Nb over-sized! ;;
out-len (.decompress decompressor ba 0 in-len ba-out* 0) len-decomp (.readInt dis)
ba-out (java.util.Arrays/copyOf ba-out* out-len)] len-comp (- (alength ^bytes ba) 4)
ba-out))) ba-comp (byte-array len-comp)
_ (.readFully dis ba-comp 0 len-comp)
ba-decomp (byte-array len-decomp)
_ (.decompress decompressor ba-comp 0 ba-decomp 0 len-decomp)]
ba-decomp)))
(def ^:private ^net.jpountz.lz4.LZ4Factory lz4-factory (def ^:private ^net.jpountz.lz4.LZ4Factory lz4-factory
(net.jpountz.lz4.LZ4Factory/fastestInstance)) (net.jpountz.lz4.LZ4Factory/fastestInstance))
@ -89,13 +97,16 @@
Write speed: very high. Write speed: very high.
Read speed: very high. Read speed: very high.
A good general-purpose compressor, competitive with Snappy." A good general-purpose compressor, competitive with Snappy.
Thanks to Max Penet (@mpenet) for our first implementation,
Ref. https://github.com/mpenet/nippy-lz4"
(->LZ4Compressor (.fastCompressor lz4-factory) (->LZ4Compressor (.fastCompressor lz4-factory)
(.safeDecompressor lz4-factory))) (.fastDecompressor lz4-factory)))
(def lz4hc-compressor "Like `lz4-compressor` but trades some speed for ratio." (def lz4hc-compressor "Like `lz4-compressor` but trades some speed for ratio."
(->LZ4Compressor (.highCompressor lz4-factory) (->LZ4Compressor (.highCompressor lz4-factory)
(.safeDecompressor lz4-factory))) (.fastDecompressor lz4-factory)))
(comment (comment
(def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8")) (def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8"))
@ -114,8 +125,5 @@
;;; 2014 April 5, initial benchmarks ;;; 2014 April 5, initial benchmarks
{:snappy {:time 2214 :ratio 0.848} {:snappy {:time 2214 :ratio 0.848}
:lzma {:time 46684 :ratio 0.494} :lzma {:time 46684 :ratio 0.494}
:lz4 {:time 1327 :ratio 0.819} ; w/o uncompressed size prefix :lz4 {:time 1363 :ratio 0.819}
:lz4hc {:time 5762 :ratio 0.763} ; '' :lz4hc {:time 6045 :ratio 0.763}})
;; :lz4 {:time 1404 :ratio 0.819} ; with uncompressed size prefix
;; :lz4hc {:time 6028 :ratio 0.763} ; ''
})