diff --git a/src/taoensso/nippy/compression.clj b/src/taoensso/nippy/compression.clj index 41e49b4..6564385 100644 --- a/src/taoensso/nippy/compression.clj +++ b/src/taoensso/nippy/compression.clj @@ -61,24 +61,32 @@ environments." (->LZMA2Compressor 0)) -(deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor - ^net.jpountz.lz4.LZ4SafeDecompressor decompressor] +(deftype LZ4Compressor [^net.jpountz.lz4.LZ4Compressor compressor + ^net.jpountz.lz4.LZ4Decompressor decompressor] ICompressor (compress [_ ba] - (let [in-len (alength ^bytes ba) - max-out-len (.maxCompressedLength compressor in-len) - ba-out* (byte-array max-out-len) - out-len (.compress compressor ba 0 in-len ba-out* 0 max-out-len) - ba-out (java.util.Arrays/copyOf ba-out* out-len)] - ba-out)) + (let [len-decomp (alength ^bytes ba) + max-len-comp (.maxCompressedLength compressor len-decomp) + ba-comp* (byte-array max-len-comp) ; Over-sized + len-comp (.compress compressor ba 0 len-decomp ba-comp* 0 max-len-comp) + ;; + baos (ByteArrayOutputStream. (+ len-comp 4)) + dos (DataOutputStream. baos)] + (.writeInt dos len-decomp) ; Prefix with uncompressed length + (.write dos ba-comp* 0 len-comp) + (.toByteArray baos))) (decompress [_ ba] - (let [in-len (alength ^bytes ba) - max-out-len in-len - ba-out* (byte-array (* max-out-len 3.0)) ; Nb over-sized! - out-len (.decompress decompressor ba 0 in-len ba-out* 0) - ba-out (java.util.Arrays/copyOf ba-out* out-len)] - ba-out))) + (let [bais (ByteArrayInputStream. ba) + dis (DataInputStream. bais) + ;; + len-decomp (.readInt dis) + len-comp (- (alength ^bytes ba) 4) + ba-comp (byte-array len-comp) + _ (.readFully dis ba-comp 0 len-comp) + ba-decomp (byte-array len-decomp) + _ (.decompress decompressor ba-comp 0 ba-decomp 0 len-decomp)] + ba-decomp))) (def ^:private ^net.jpountz.lz4.LZ4Factory lz4-factory (net.jpountz.lz4.LZ4Factory/fastestInstance)) @@ -89,13 +97,16 @@ Write speed: very high. Read speed: very high. - A good general-purpose compressor, competitive with Snappy." + A good general-purpose compressor, competitive with Snappy. + + Thanks to Max Penet (@mpenet) for our first implementation, + Ref. https://github.com/mpenet/nippy-lz4" (->LZ4Compressor (.fastCompressor lz4-factory) - (.safeDecompressor lz4-factory))) + (.fastDecompressor lz4-factory))) (def lz4hc-compressor "Like `lz4-compressor` but trades some speed for ratio." (->LZ4Compressor (.highCompressor lz4-factory) - (.safeDecompressor lz4-factory))) + (.fastDecompressor lz4-factory))) (comment (def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8")) @@ -114,8 +125,5 @@ ;;; 2014 April 5, initial benchmarks {:snappy {:time 2214 :ratio 0.848} :lzma {:time 46684 :ratio 0.494} - :lz4 {:time 1327 :ratio 0.819} ; w/o uncompressed size prefix - :lz4hc {:time 5762 :ratio 0.763} ; '' - ;; :lz4 {:time 1404 :ratio 0.819} ; with uncompressed size prefix - ;; :lz4hc {:time 6028 :ratio 0.763} ; '' - }) + :lz4 {:time 1363 :ratio 0.819} + :lz4hc {:time 6045 :ratio 0.763}})