luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[new] [#164] Update benchmarks

Browse source
This commit is contained in:
Peter Taoussanis 2024-01-15 12:45:21 +01:00
parent f287df9e9c
commit dcc6b081f1
4 changed files with 105 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
README.md
View file

@ -38,15 +38,10 @@ See [here][GitHub releases] for earlier releases.
## Performance ## Performance
Since its earliest versions, Nippy has consistently been the **fastest serialization library for Clojure** that I'm aware of. It offers: Since its earliest versions, Nippy has consistently been the **fastest serialization library for Clojure** that I'm aware of. Latest [benchmark](../../blob/master/test/taoensso/nippy_benchmarks.clj) results:
- Roundtrip times **>12x faster** than `tools.reader` with **60% smaller** data size.
- Roundtrip times **>2x faster** than `data.fressian` with **30% smaller** data size.
![benchmarks-png](../../raw/master/benchmarks.png) ![benchmarks-png](../../raw/master/benchmarks.png)
The [benchmark code](../../blob/master/test/taoensso/nippy_benchmarks.clj) can be easily run in your own environment.
## Documentation ## Documentation
- [Wiki][GitHub wiki] (getting started, usage, etc.) - [Wiki][GitHub wiki] (getting started, usage, etc.)

BIN
benchmarks.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 127 KiB

After

Width:  |  Height:  |  Size: 68 KiB

174
test/taoensso/nippy_benchmarks.clj
View file

@ -25,9 +25,9 @@
(comment (-> data freeze-fress thaw-fress)) (comment (-> data freeze-fress thaw-fress))
;;;; Benchable data ;;;; Bench data
(def bench-data (def default-bench-data
"Subset of stress data suitable for benching." "Subset of stress data suitable for benching."
(let [sd (nippy/stress-data {:comparable? true})] (let [sd (nippy/stress-data {:comparable? true})]
(reduce-kv (reduce-kv
@ -42,12 +42,18 @@
(comment (comment
(clojure.set/difference (clojure.set/difference
(set (keys (nippy/stress-data {:comparable? true}))) (set (keys (nippy/stress-data {:comparable? true})))
(set (keys bench-data)))) (set (keys default-bench-data))))
;;;; ;;;; Serialization
(defn- bench1-serialization
[freezer thawer sizer
{:keys [laps warmup bench-data]
:or
{laps 1e4
warmup 25e3
bench-data default-bench-data}}]
(defn- bench1
[{:keys [laps warmup] :or {laps 1e4, warmup 25e3}} freezer thawer sizer]
(let [data-frozen (freezer bench-data) (let [data-frozen (freezer bench-data)
time-freeze (enc/bench laps {:warmup-laps warmup} (freezer bench-data)) time-freeze (enc/bench laps {:warmup-laps warmup} (freezer bench-data))
time-thaw (enc/bench laps {:warmup-laps warmup} (thawer data-frozen)) time-thaw (enc/bench laps {:warmup-laps warmup} (thawer data-frozen))
@ -58,90 +64,90 @@
:thaw time-thaw :thaw time-thaw
:size data-size})) :size data-size}))
(comment (bench1 {} nippy/freeze nippy/thaw count)) (comment (bench1-serialization nippy/freeze nippy/thaw count {}))
(defn bench (defn- printed-results [results]
[{:keys [all? reader? fressian? fressian? lzma2? laps warmup] (println "\nBenchmark results:")
:or {laps 1e4 (doseq [[k v] results] (println k " " v))
warmup 25e3}}] (do results))
(defn bench-serialization
[{:keys [all? reader? fressian? fressian? lzma2? laps warmup bench-data]
:as opts
:or
{laps 1e4
warmup 25e3}}]
(println "\nStarting benchmarks") (println "\nStarting benchmarks")
(let [results_ (atom {})] (let [results_ (atom {})]
(when (or all? reader?) (when (or all? reader?)
(println "Benching Reader...") (println "- Benching Reader...")
(swap! results_ assoc :reader (swap! results_ assoc :reader
(bench1 {:laps laps, :warmup warmup} (bench1-serialization freeze-reader thaw-reader
freeze-reader thaw-reader (fn [^String s] (count (.getBytes s "UTF-8")))
(fn [^String s] (count (.getBytes s "UTF-8")))))) (assoc opts :laps laps, :warmup warmup))))
(when (or all? fressian?) (when (or all? fressian?)
(println "- Benching Fressian...") (println "- Benching Fressian...")
(swap! results_ assoc :fressian (bench1 {:laps laps, :warmup warmup} freeze-fress thaw-fress count))) (swap! results_ assoc :fressian
(bench1-serialization freeze-fress thaw-fress count
(assoc opts :laps laps, :warmup warmup))))
(when (or all? lzma2?) (when (or all? lzma2?)
(println "- Benching Nippy/LZMA2...") (println "- Benching Nippy/LZMA2...")
(swap! results_ assoc :nippy/lzma2 (swap! results_ assoc :nippy/lzma2
(bench1 {:laps laps, :warmup warmup} (bench1-serialization
#(nippy/freeze % {:compressor nippy/lzma2-compressor}) #(nippy/freeze % {:compressor nippy/lzma2-compressor})
#(nippy/thaw % {:compressor nippy/lzma2-compressor}) #(nippy/thaw % {:compressor nippy/lzma2-compressor})
count))) count
(assoc opts :laps laps, :warmup warmup))))
(println "- Benching Nippy/encrypted...") (println "- Benching Nippy/encrypted...")
(swap! results_ assoc :nippy/encrypted (swap! results_ assoc :nippy/encrypted
(bench1 {:laps laps, :warmup warmup} (bench1-serialization
#(nippy/freeze % {:password [:cached "p"]}) #(nippy/freeze % {:password [:cached "p"]})
#(nippy/thaw % {:password [:cached "p"]}) #(nippy/thaw % {:password [:cached "p"]})
count)) count
(assoc opts :laps laps, :warmup warmup)))
(println "- Benching Nippy/default...") (println "- Benching Nippy/default...")
(swap! results_ assoc :nippy/default (bench1 {:laps laps, :warmup warmup} nippy/freeze nippy/thaw count)) (swap! results_ assoc :nippy/default
(bench1-serialization nippy/freeze nippy/thaw count
(assoc opts :laps laps, :warmup warmup)))
(println "- Benching Nippy/fast...") (println "- Benching Nippy/fast...")
(swap! results_ assoc :nippy/fast (bench1 {:laps laps, :warmup warmup} nippy/fast-freeze nippy/fast-thaw count)) (swap! results_ assoc :nippy/fast
(bench1-serialization nippy/fast-freeze nippy/fast-thaw count
(assoc opts :laps laps, :warmup warmup)))
(println "- Benchmarks complete! (Time for cake?)") (println "- Benchmarks complete! (Time for cake?)")
(printed-results @results_)))
(let [results @results_] ;;;; Compression
(println "\nBenchmark results:")
(doseq [[k v] results] (println k " " v))
(do results))))
(comment (defn- bench1-compressor
(do [compressor
(set! *unchecked-math* false) {:keys [laps warmup bench-data]
(bench {:all? true})) :or
{laps 1e4
warmup 2e4
bench-data default-bench-data}}]
;; 2023 Sep 25, 2020 Apple MBP M1 (let [data-frozen (nippy/freeze bench-data {:compressor nil})
;; [com.taoensso/nippy "3.2.0"] data-compressed (compr/compress compressor data-frozen)
;; [org.clojure/tools.reader "1.3.6"] time-compress (enc/bench laps {:warmup-laps warmup} (compr/compress compressor data-frozen))
;; [org.clojure/data.fressian "1.0.0"] time-decompress (enc/bench laps {:warmup-laps warmup} (compr/decompress compressor data-compressed))]
{:reader {:round 36756, :freeze 6567, :thaw 30189, :size 39374} {:round (+ time-compress time-decompress)
:fressian {:round 6579, :freeze 3720, :thaw 2859, :size 22850} :compress time-compress
:nippy/lzma2 {:round 33645, :freeze 20411, :thaw 13234, :size 11420} :decompress time-decompress
:nippy/encrypted {:round 3894, :freeze 2260, :thaw 1634, :size 16717} :ratio (enc/round2 (/ (count data-compressed) (count data-frozen)))}))
:nippy/default {:round 2796, :freeze 1467, :thaw 1329, :size 16689}
:nippy/fast {:round 2516, :freeze 1271, :thaw 1245, :size 28454}})
;;;; Compressors (defn bench-compressors [opts lzma-opts]
(printed-results
(let [bench-data (nippy/freeze (nippy/stress-data {:comparable? true}) {:compressor nil})]
(defn bench1-compressor
[{:keys [laps warmup] :or {laps 1e4, warmup 2e4}} compressor]
(let [data-compressed (compr/compress compressor bench-data)
time-compress (enc/bench laps {:warmup-laps warmup} (compr/compress compressor bench-data))
time-decompress (enc/bench laps {:warmup-laps warmup} (compr/decompress compressor data-compressed))]
{:round (+ time-compress time-decompress)
:compress time-compress
:decompress time-decompress
:ratio (enc/round2 (/ (count data-compressed) (count bench-data)))}))
(defn bench-compressors [bench1-opts lzma-opts]
(merge (merge
(let [bench1 #(bench1-compressor bench1-opts %)] (let [bench1 #(bench1-compressor % opts)]
{:zstd/prepended (bench1 (compr/->ZstdCompressor true)) {:zstd/prepended (bench1 (compr/->ZstdCompressor true))
:zstd/unprepended (bench1 (compr/->ZstdCompressor false)) :zstd/unprepended (bench1 (compr/->ZstdCompressor false))
:lz4 (bench1 (compr/->LZ4Compressor)) :lz4 (bench1 (compr/->LZ4Compressor))
@ -149,28 +155,52 @@
:snappy/prepended (bench1 (compr/->SnappyCompressor true)) :snappy/prepended (bench1 (compr/->SnappyCompressor true))
:snappy/unprepended (bench1 (compr/->SnappyCompressor false))}) :snappy/unprepended (bench1 (compr/->SnappyCompressor false))})
(let [bench1 #(bench1-compressor (merge bench1-opts lzma-opts) %)] (let [bench1 #(bench1-compressor % (merge opts lzma-opts))]
{:lzma2/level0 (bench1 (compr/->LZMA2Compressor 0)) {:lzma2/level0 (bench1 (compr/->LZMA2Compressor 0))
:lzma2/level3 (bench1 (compr/->LZMA2Compressor 3)) :lzma2/level3 (bench1 (compr/->LZMA2Compressor 3))
:lzma2/level6 (bench1 (compr/->LZMA2Compressor 6)) :lzma2/level6 (bench1 (compr/->LZMA2Compressor 6))
:lzma2/level9 (bench1 (compr/->LZMA2Compressor 9))})))) :lzma2/level9 (bench1 (compr/->LZMA2Compressor 9))}))))
;;;; Results
(comment (comment
{:last-updated "2024-01-16"
:clojure-version "1.11.1"
:java-version "OpenJDK 21"
:deps
'[[com.taoensso/nippy "3.4.0-beta1"]
[org.clojure/tools.reader "1.3.7"]
[org.clojure/data.fressian "1.0.0"]
[org.tukaani/xz "1.9"]
[io.airlift/aircompressor "0.25"]]}
(bench-serialization {:all? true})
{:reader {:round 13496, :freeze 5088, :thaw 8408, :size 15880}
:fressian {:round 3898, :freeze 2350, :thaw 1548, :size 12222}
:nippy/lzma2 {:round 12341, :freeze 7809, :thaw 4532, :size 3916}
:nippy/encrypted {:round 2939, :freeze 1505, :thaw 1434, :size 8547}
:nippy/default {:round 2704, :freeze 1330, :thaw 1374, :size 8519}
:nippy/fast {:round 2425, :freeze 1117, :thaw 1308, :size 17088}}
(enc/round2 (/ 2704 13496)) ; 0.20 of reader roundtrip time
(enc/round2 (/ 2704 3898)) ; 0.69 of fressian roundtrip time
(enc/round2 (/ 8519 15880)) ; 0.54 of reader output size
(enc/round2 (/ 8519 12222)) ; 0.70 of fressian output size
(bench-compressors (bench-compressors
{:laps 1e4 :warmup 2e4} {:laps 1e4 :warmup 2e4}
{:laps 1e2 :warmup 2e2}) {:laps 1e2 :warmup 2e2})
;; 2023 Aug 1, 2020 Apple MBP M1 ;; Note that ratio depends on compressibility of stress data
;; [org.tukaani/xz "1.9"] {:lz4 {:round 293, :compress 234, :decompress 59, :ratio 0.5}
;; [io.airlift/aircompressor "0.25"] :lzo {:round 483, :compress 349, :decompress 134, :ratio 0.46}
:snappy/prepended {:round 472, :compress 296, :decompress 176, :ratio 0.43}
{:zstd/prepended {:round 1672, :compress 1279, :decompress 393, :ratio 0.53} :snappy/unprepended {:round 420, :compress 260, :decompress 160, :ratio 0.43}
:zstd/unprepended {:round 1668, :compress 1271, :decompress 397, :ratio 0.53} :zstd/prepended {:round 2105, :compress 1419, :decompress 686, :ratio 0.3}
:lz4 {:round 269, :compress 238, :decompress 31, :ratio 0.58} :zstd/unprepended {:round 1261, :compress 921, :decompress 340, :ratio 0.3}
:lzo {:round 259, :compress 216, :decompress 43, :ratio 0.58} :lzma2/level0 {:round 158, :compress 121, :decompress 37, :ratio 0.24}
:snappy/prepended {:round 339, :compress 205, :decompress 134, :ratio 0.58} :lzma2/level3 {:round 536, :compress 436, :decompress 100, :ratio 0.22}
:snappy/unprepended {:round 340, :compress 206, :decompress 134, :ratio 0.58} :lzma2/level6 {:round 1136, :compress 1075, :decompress 61, :ratio 0.21}
:lzma2/level0 {:round 30300, :compress 18500, :decompress 11800, :ratio 0.4} :lzma2/level9 {:round 2391, :compress 2096, :decompress 295, :ratio 0.21}})
:lzma2/level3 {:round 49200, :compress 35700, :decompress 13500, :ratio 0.4}
:lzma2/level6 {:round 102600, :compress 86700, :decompress 15900, :ratio 0.41}
:lzma2/level9 {:round 434800, :compress 394700, :decompress 40100, :ratio 0.41}})

3
test/taoensso/nippy_tests.clj
View file

@ -436,4 +436,5 @@
;;;; Benchmarks ;;;; Benchmarks
(deftest _benchmarks (is (benchmarks/bench {}))) (deftest _benchmarks
(is (benchmarks/bench-serialization {:all? true})))