luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[nop] [#176] make deps on compression libs optional

Browse source 
Use reflective calls to interact with org.tukaani/xz and io.airlift/aircompressor so that users of nippy can choose
to exclude those dependencies if they aren't using compression.

Reflective calls are much slower, and some type-hints were also removed, so these changes might affect performance.
This commit is contained in:
Tom Denley 2024-06-06 15:15:17 +01:00
parent 5a9a391331
commit ba366e5294
1 changed files with 43 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
src/taoensso/nippy/compression.clj
View file

@ -5,7 +5,7 @@
[java.nio ByteBuffer] [java.nio ByteBuffer]
[java.io [java.io
ByteArrayInputStream ByteArrayOutputStream ByteArrayInputStream ByteArrayOutputStream
DataInputStream DataOutputStream])) DataInputStream DataOutputStream InputStream OutputStream]))
;;;; Interface ;;;; Interface
@ -37,7 +37,7 @@
;;;; Airlift ;;;; Airlift
(defn- airlift-compress (defn- airlift-compress
^bytes [^io.airlift.compress.Compressor c ^bytes ba prepend-size?] ^bytes [c ^bytes ba prepend-size?]
(let [in-len (alength ba) (let [in-len (alength ba)
max-out-len (.maxCompressedLength c in-len)] max-out-len (.maxCompressedLength c in-len)]
@ -65,7 +65,7 @@
(java.util.Arrays/copyOfRange ba-max-out 0 out-len)))))) (java.util.Arrays/copyOfRange ba-max-out 0 out-len))))))
(defn- airlift-decompress (defn- airlift-decompress
^bytes [^io.airlift.compress.Decompressor d ^bytes ba max-out-len] ^bytes [d ^bytes ba max-out-len]
(if max-out-len (if max-out-len
(let [max-out-len (int max-out-len) (let [max-out-len (int max-out-len)
ba-max-out (byte-array max-out-len) ba-max-out (byte-array max-out-len)
@ -86,21 +86,36 @@
ba-out 0 out-len) ba-out 0 out-len)
ba-out))) ba-out)))
(defn- new-instance
([s]
(new-instance s [] []))
([s types args]
(.newInstance (.getDeclaredConstructor (Class/forName s) (into-array Class types))
(into-array Object args))))
(do (do
(enc/def* ^:private airlift-zstd-compressor_ (enc/thread-local (io.airlift.compress.zstd.ZstdCompressor.))) (enc/def* ^:private airlift-zstd-compressor_ (enc/thread-local
(enc/def* ^:private airlift-zstd-decompressor_ (enc/thread-local (io.airlift.compress.zstd.ZstdDecompressor.))) (new-instance "io.airlift.compress.zstd.ZstdCompressor")))
(enc/def* ^:private airlift-zstd-decompressor_ (enc/thread-local
(new-instance "io.airlift.compress.zstd.ZstdDecompressor")))
(deftype ZstdCompressor [prepend-size?] (deftype ZstdCompressor [prepend-size?]
ICompressor ICompressor
(header-id [_] :zstd) (header-id [_] :zstd)
(compress [_ ba] (airlift-compress @airlift-zstd-compressor_ ba prepend-size?)) (compress [_ ba] (airlift-compress @airlift-zstd-compressor_ ba prepend-size?))
(decompress [_ ba] (airlift-decompress @airlift-zstd-decompressor_ ba (decompress [_ ba] (airlift-decompress @airlift-zstd-decompressor_ ba
(when-not prepend-size? (when-not prepend-size?
(io.airlift.compress.zstd.ZstdDecompressor/getDecompressedSize ba (.invoke
0 (alength ^bytes ba))))))) (.getDeclaredMethod (Class/forName "io.airlift.compress.zstd.ZstdDecompressor")
"getDecompressedSize"
(into-array Class [(Class/forName "[B") Integer/TYPE Integer/TYPE]))
nil
(into-array Object [ba (int 0) (alength ^bytes ba)])))))))
(do (do
(enc/def* ^:private airlift-lz4-compressor_ (enc/thread-local (io.airlift.compress.lz4.Lz4Compressor.))) (enc/def* ^:private airlift-lz4-compressor_ (enc/thread-local
(enc/def* ^:private airlift-lz4-decompressor_ (enc/thread-local (io.airlift.compress.lz4.Lz4Decompressor.))) (new-instance "io.airlift.compress.lz4.Lz4Compressor")))
(enc/def* ^:private airlift-lz4-decompressor_ (enc/thread-local
(new-instance "io.airlift.compress.lz4.Lz4Decompressor")))
(deftype LZ4Compressor [] (deftype LZ4Compressor []
ICompressor ICompressor
(header-id [_] :lz4) (header-id [_] :lz4)
@ -108,8 +123,10 @@
(decompress [_ ba] (airlift-decompress @airlift-lz4-decompressor_ ba nil)))) (decompress [_ ba] (airlift-decompress @airlift-lz4-decompressor_ ba nil))))
(do (do
(enc/def* ^:private airlift-lzo-compressor_ (enc/thread-local (io.airlift.compress.lzo.LzoCompressor.))) (enc/def* ^:private airlift-lzo-compressor_ (enc/thread-local
(enc/def* ^:private airlift-lzo-decompressor_ (enc/thread-local (io.airlift.compress.lzo.LzoDecompressor.))) (new-instance "io.airlift.compress.lzo.LzoCompressor")))
(enc/def* ^:private airlift-lzo-decompressor_ (enc/thread-local
(new-instance "io.airlift.compress.lzo.LzoDecompressor")))
(deftype LZOCompressor [] (deftype LZOCompressor []
ICompressor ICompressor
(header-id [_] :lzo) (header-id [_] :lzo)
@ -117,15 +134,22 @@
(decompress [_ ba] (airlift-decompress @airlift-lzo-decompressor_ ba nil)))) (decompress [_ ba] (airlift-decompress @airlift-lzo-decompressor_ ba nil))))
(do (do
(enc/def* ^:private airlift-snappy-compressor_ (enc/thread-local (io.airlift.compress.snappy.SnappyCompressor.))) (enc/def* ^:private airlift-snappy-compressor_ (enc/thread-local
(enc/def* ^:private airlift-snappy-decompressor_ (enc/thread-local (io.airlift.compress.snappy.SnappyDecompressor.))) (new-instance "io.airlift.compress.snappy.SnappyCompressor")))
(enc/def* ^:private airlift-snappy-decompressor_ (enc/thread-local
(new-instance "io.airlift.compress.snappy.SnappyDecompressor")))
(deftype SnappyCompressor [prepend-size?] (deftype SnappyCompressor [prepend-size?]
ICompressor ICompressor
(header-id [_] :snappy) (header-id [_] :snappy)
(compress [_ ba] (airlift-compress @airlift-snappy-compressor_ ba prepend-size?)) (compress [_ ba] (airlift-compress @airlift-snappy-compressor_ ba prepend-size?))
(decompress [_ ba] (airlift-decompress @airlift-snappy-decompressor_ ba (decompress [_ ba] (airlift-decompress @airlift-snappy-decompressor_ ba
(when-not prepend-size? (when-not prepend-size?
(io.airlift.compress.snappy.SnappyDecompressor/getUncompressedLength ba 0)))))) (.invoke
(.getDeclaredMethod (Class/forName "io.airlift.compress.snappy.SnappyDecompressor")
"getUncompressedLength"
(into-array Class [(Class/forName "[B") Integer/TYPE]))
nil
(into-array Object [ba (int 0)])))))))
;;;; LZMA2 ;;;; LZMA2
@ -140,8 +164,10 @@
len-decomp (alength ^bytes ba) len-decomp (alength ^bytes ba)
;; Prefix with uncompressed length: ;; Prefix with uncompressed length:
_ (.writeInt dos len-decomp) _ (.writeInt dos len-decomp)
xzs (org.tukaani.xz.XZOutputStream. baos filter-options (new-instance "org.tukaani.xz.LZMA2Options" [Integer/TYPE] [(int compression-level)])
(org.tukaani.xz.LZMA2Options. compression-level))] xzs (new-instance "org.tukaani.xz.XZOutputStream"
[OutputStream (Class/forName "org.tukaani.xz.FilterOptions")]
[baos filter-options])]
(.write xzs ^bytes ba) (.write xzs ^bytes ba)
(.close xzs) (.close xzs)
(.toByteArray baos))) (.toByteArray baos)))
@ -152,7 +178,7 @@
;; ;;
len-decomp (.readInt dis) len-decomp (.readInt dis)
ba (byte-array len-decomp) ba (byte-array len-decomp)
xzs (org.tukaani.xz.XZInputStream. bais)] xzs (new-instance "org.tukaani.xz.XZInputStream" [InputStream] [bais])]
(.read xzs ba 0 len-decomp) (.read xzs ba 0 len-decomp)
(if (== -1 (.read xzs)) ; Good practice as extra safety measure (if (== -1 (.read xzs)) ; Good practice as extra safety measure
nil nil