diff --git a/benchmarks.png b/benchmarks.png index af74558..1c2a617 100644 Binary files a/benchmarks.png and b/benchmarks.png differ diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 942ab43..c669aff 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -2103,15 +2103,19 @@ :ex-info (ex-info "ExInfo" {:data "data"})}) (def stress-data-comparable - "Reference data with stuff removed that breaks roundtrip equality" - (dissoc stress-data :bytes :throwable :exception :ex-info :regex :objects)) + "Reference data with stuff removed that breaks roundtrip equality." + (dissoc stress-data :bytes :objects :throwable :exception :ex-info :regex)) + +(comment (let [data stress-data-comparable] (= (thaw (freeze data)) data))) (def stress-data-benchable "Reference data with stuff removed that breaks reader or other utils we'll - be benching against" - (dissoc stress-data - :bytes :throwable :exception :ex-info :queue :queue-empty - :byte :stress-record :stress-type :regex :objects)) + be benching with." + (dissoc stress-data-comparable + :queue :queue-empty + :stress-record :stress-type + :time-instant :time-duration :time-period + :byte :uri)) ;;;; Tools diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 21b9ed3..d851b33 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,168 +1,127 @@ (ns taoensso.nippy.benchmarks "Nippy benchmarks." - (:require [clojure.data.fressian :as fressian] - [taoensso.encore :as enc] - [taoensso.nippy :as nippy :refer [freeze thaw]])) + (:require + [clojure.data.fressian :as fress] + [taoensso.encore :as enc] + [taoensso.nippy :as nippy])) -(def data #_22 nippy/stress-data-benchable) +;;;; Reader -(defn fressian-freeze [value] - (let [^java.nio.ByteBuffer bb (fressian/write value) +(defn- freeze-reader [x] (enc/pr-edn x)) +(defn- thaw-reader [edn] (enc/read-edn edn)) + +;;;; Fressian + +(defn- freeze-fress [x] + (let [^java.nio.ByteBuffer bb (fress/write x) len (.remaining bb) ba (byte-array len)] (.get bb ba 0 len) - ba)) + (do ba))) -(defn fressian-thaw [value] - (let [bb (java.nio.ByteBuffer/wrap value)] - (fressian/read bb))) +(defn- thaw-fress [^bytes ba] + (let [bb (java.nio.ByteBuffer/wrap ba)] + (fress/read bb))) -(comment (fressian-thaw (fressian-freeze data))) +(comment (-> data freeze-fress thaw-fress)) -(defmacro bench* [& body] `(enc/bench 10000 {:warmup-laps 25000} ~@body)) -(defn bench1 [freezer thawer & [sizer]] - (let [data-frozen (freezer data) - time-freeze (bench* (freezer data)) - time-thaw (bench* (thawer data-frozen))] - {:round (+ time-freeze time-thaw) - :freeze time-freeze - :thaw time-thaw - :size ((or sizer count) data-frozen)})) +;;;; Benchable data -(defn bench [{:keys [reader? lzma2? fressian? laps] :or {laps 1}}] - (println "\nBenching (this can take some time)") - (println "----------------------------------") - (dotimes [l laps] - (println (str "\nLap " (inc l) "/" laps "...")) - - (when reader? ; Slow - (println {:reader (bench1 enc/pr-edn enc/read-edn - #(count (.getBytes ^String % "UTF-8")))})) - - (when lzma2? ; Slow - (println {:lzma2 (bench1 #(freeze % {:compressor nippy/lzma2-compressor}) - #(thaw % {:compressor nippy/lzma2-compressor}))})) - - (when fressian? - (println {:fressian (bench1 fressian-freeze fressian-thaw)})) - - (println {:encrypted (bench1 #(freeze % {:password [:cached "p"]}) - #(thaw % {:password [:cached "p"]}))}) - (println {:default (bench1 #(freeze % {}) - #(thaw % {}))}) - (println {:fast (bench1 nippy/fast-freeze nippy/fast-thaw)})) - - (println "\nDone! (Time for cake?)") - true) - -(comment (enc/read-edn (enc/pr-edn data)) - (bench1 fressian-freeze fressian-thaw)) +(def data + "Map of data suitable for benching, a subset of + `nippy/stress-data-comparable`." + (reduce-kv + (fn [m k v] + (try + (-> v freeze-reader thaw-reader) + (-> v freeze-fress thaw-fress) + m + (catch Throwable _ (dissoc m k)))) + nippy/stress-data-comparable + nippy/stress-data-comparable)) (comment - (set! *unchecked-math* false) - ;; (bench {:reader? true :lzma2? true :fressian? true :laps 2}) - ;; (bench {:laps 2}) + (clojure.set/difference + (set (keys nippy/stress-data-comparable)) + (set (keys data)))) - ;;; 2016 Jul 17, v2.12.0-RC2, minor final optimizations - {:encrypted {:round 4527, :freeze 2651, :thaw 1876, :size 16324}} - {:default {:round 3998, :freeze 2226, :thaw 1772, :size 16297}} - {:fast {:round 3408, :freeze 1745, :thaw 1663, :size 17069}} +;;;; - ;;; 2016 Apr 14, v2.12.0-SNAPSHOT, refactor + larger data + new hardware - {:reader {:round 52380, :freeze 17817, :thaw 34563, :size 27861}} - {:lzma2 {:round 43321, :freeze 28312, :thaw 15009, :size 11260}} - {:fressian {:round 6911, :freeze 5109, :thaw 1802, :size 17105}} - {:encrypted {:round 4726, :freeze 2951, :thaw 1775, :size 16308}} - {:default {:round 4299, :freeze 2655, :thaw 1644, :size 16278}} - {:fast {:round 3739, :freeze 2159, :thaw 1580, :size 17069}} - ;; 12.184228890439638 :default - ;; 14.009093340465364 :fast +(defn- bench1 + [{:keys [laps warmup] :or {laps 1e4, warmup 25e3}} freezer thawer sizer] + (let [data-frozen (freezer data) + time-freeze (enc/bench laps {:warmup-laps warmup} (freezer data)) + time-thaw (enc/bench laps {:warmup-laps warmup} (thawer data-frozen)) + data-size (sizer data-frozen)] - ;;; 2015 Oct 6, v2.11.0-alpha4 - {:reader {:round 73409, :freeze 21823, :thaw 51586, :size 27672}} - {:lzma2 {:round 56689, :freeze 37222, :thaw 19467, :size 11252}} - {:fressian {:round 10666, :freeze 7737, :thaw 2929, :size 16985}} - {:encrypted {:round 6885, :freeze 4227, :thaw 2658, :size 16148}} - {:default {:round 6304, :freeze 3824, :thaw 2480, :size 16122}} - {:fast1 {:round 5352, :freeze 3272, :thaw 2080, :size 16976}} - {:fast2 {:round 5243, :freeze 3238, :thaw 2005, :size 16972}} - ;; :reader/:default ratio: 11.64 - ;; - {:reader {:round 26, :freeze 17, :thaw 9, :size 2}} - {:lzma2 {:round 3648, :freeze 3150, :thaw 498, :size 68}} - {:fressian {:round 19, :freeze 7, :thaw 12, :size 1}} - {:encrypted {:round 63, :freeze 40, :thaw 23, :size 36}} - {:default {:round 24, :freeze 17, :thaw 7, :size 6}} - {:fast1 {:round 19, :freeze 12, :thaw 7, :size 6}} - {:fast2 {:round 4, :freeze 2, :thaw 2, :size 2}} + {:round (+ time-freeze time-thaw) + :freeze time-freeze + :thaw time-thaw + :size data-size})) - ;;; 2015 Sep 29, after read/write API refactor - {:lzma2 {:round 51640, :freeze 33699, :thaw 17941, :size 11240}} - {:encrypted {:round 5922, :freeze 3734, :thaw 2188, :size 16132}} - {:default {:round 5588, :freeze 3658, :thaw 1930, :size 16113}} - {:fast {:round 4533, :freeze 2688, :thaw 1845, :size 16972}} +(comment (bench1 {} nippy/freeze nippy/thaw count)) - ;;; 2015 Sep 28, small collection optimizations - {:lzma2 {:round 56307, :freeze 36475, :thaw 19832, :size 11244}} - {:encrypted {:round 6062, :freeze 3802, :thaw 2260, :size 16148}} - {:default {:round 5482, :freeze 3382, :thaw 2100, :size 16128}} - {:fast {:round 4729, :freeze 2826, :thaw 1903, :size 16972}} +(defn bench + [{:keys [all? reader? fressian? fressian? lzma2? laps warmup] + :or {laps 1e4 + warmup 25e3}}] - ;;; 2015 Sep 29, various micro optimizations (incl. &arg elimination) - {:reader {:round 63547, :freeze 19374, :thaw 44173, :size 27717}} - {:lzma2 {:round 51724, :freeze 33502, :thaw 18222, :size 11248}} - {:fressian {:round 8813, :freeze 6460, :thaw 2353, :size 16985}} - {:encrypted {:round 6005, :freeze 3768, :thaw 2237, :size 16164}} - {:default {:round 5417, :freeze 3354, :thaw 2063, :size 16145}} - {:fast {:round 4659, :freeze 2712, :thaw 1947, :size 17026}} + (println "\nStarting benchmarks") + (println "---") - ;;; 2015 Sep 15 - v2.10.0-alpha6, Clojure 1.7.0 - {:reader {:round 94901, :freeze 25781, :thaw 69120, :size 27686}} - {:lzma2 {:round 65127, :freeze 43150, :thaw 21977, :size 11244}} - {:encrypted {:round 12590, :freeze 7565, :thaw 5025, :size 16148}} - {:fressian {:round 12085, :freeze 9168, :thaw 2917, :size 16972}} - {:default {:round 6974, :freeze 4582, :thaw 2392, :size 16123}} - {:fast {:round 6255, :freeze 3724, :thaw 2531, :size 17013}} + (let [results_ (atom {})] + (when (or all? reader?) + (println "Benching Reader...") + (swap! results_ assoc :reader + (bench1 {:laps laps, :warmup warmup} + freeze-reader thaw-reader + (fn [^String s] (count (.getBytes s "UTF-8")))))) - ;;; 2015 Sep 14 - v2.10.0-alpha5, Clojure 1.7.0-RC1 - {:default {:round 6870, :freeze 4376, :thaw 2494, :size 16227}} - {:fast {:round 6104, :freeze 3743, :thaw 2361, :size 17013}} - {:encrypted {:round 12155, :freeze 6908, :thaw 5247, :size 16244}} + (when (or all? fressian?) + (println "- Benching Fressian...") + (swap! results_ assoc :fressian (bench1 {:laps laps, :warmup warmup} freeze-fress thaw-fress count))) - ;;; 2015 June 4 - v2.9.0, Clojure 1.7.0-RC1 - {:reader {:round 155353, :freeze 44192, :thaw 111161, :size 27693}} - {:lzma2 {:round 102484, :freeze 68274, :thaw 34210, :size 11240}} - {:fressian {:round 44665, :freeze 34996, :thaw 9669, :size 16972}} - {:encrypted {:round 19791, :freeze 11354, :thaw 8437, :size 16148}} - {:default {:round 12302, :freeze 8310, :thaw 3992, :size 16126}} - {:fast {:round 9802, :freeze 5944, :thaw 3858, :size 17013}} + (when (or all? lzma2?) + (println "- Benching Nippy/LZMA2...") + (swap! results_ assoc :nippy/lzma2 + (bench1 {:laps laps, :warmup warmup} + #(nippy/freeze % {:compressor nippy/lzma2-compressor}) + #(nippy/thaw % {:compressor nippy/lzma2-compressor}) + count))) - ;;; 2015 Apr 17 w/ smart compressor selection, Clojure 1.7.0-beta1 - {:default {:round 6163, :freeze 4095, :thaw 2068, :size 16121}} - {:fast {:round 5417, :freeze 3480, :thaw 1937, :size 17013}} - {:encrypted {:round 10950, :freeze 6400, :thaw 4550, :size 16148}} + (println "- Benching Nippy/encrypted...") + (swap! results_ assoc :nippy/encrypted + (bench1 {:laps laps, :warmup warmup} + #(nippy/freeze % {:password [:cached "p"]}) + #(nippy/thaw % {:password [:cached "p"]}) + count)) - ;;; 2014 Apr 7 w/ some additional implementation tuning - {:default {:round 6533, :freeze 3618, :thaw 2915, :size 16139}} - {:fast {:round 6250, :freeze 3376, :thaw 2874, :size 16992}} - {:encrypted {:round 10583, :freeze 5581, :thaw 5002, :size 16164}} + (println "- Benching Nippy/default...") + (swap! results_ assoc :nippy/default (bench1 {:laps laps, :warmup warmup} nippy/freeze nippy/thaw count)) - ;;; 2014 Apr 5 w/ headerless :fast, LZ4 replacing Snappy as default compressor - {:default {:round 7039, :freeze 3865, :thaw 3174, :size 16123}} - {:fast {:round 6394, :freeze 3379, :thaw 3015, :size 16992}} - {:encrypted {:round 11035, :freeze 5860, :thaw 5175, :size 16148}} + (println "- Benching Nippy/fast...") + (swap! results_ assoc :nippy/fast (bench1 {:laps laps, :warmup warmup} nippy/fast-freeze nippy/fast-thaw count)) - ;;; 2014 Jan 22: with common-type size optimizations, enlarged stress-data - {:reader {:round 109544, :freeze 39523, :thaw 70021, :size 27681}} - {:default {:round 9234, :freeze 5128, :thaw 4106, :size 15989}} - {:fast {:round 7402, :freeze 4021, :thaw 3381, :size 16957}} - {:encrypted {:round 12594, :freeze 6884, :thaw 5710, :size 16020}} - {:lzma2 {:round 66759, :freeze 44246, :thaw 22513, :size 11208}} - {:fressian {:round 13052, :freeze 8694, :thaw 4358, :size 16942}} + (println "- Benchmarks complete! (Time for cake?)") - ;;; 19 Oct 2013: Nippy v2.3.0, with lzma2 & (nb!) round=freeze+thaw - {:reader {:round 67798, :freeze 23202, :thaw 44596, :size 22971}} - {:default {:round 3632, :freeze 2349, :thaw 1283, :size 12369}} - {:encrypted {:round 6970, :freeze 4073, :thaw 2897, :size 12388}} - {:fast {:round 3294, :freeze 2109, :thaw 1185, :size 13277}} - {:lzma2 {:round 44590, :freeze 29567, :thaw 15023, :size 9076}}) + (let [results @results_] + (println "\n") + (doseq [[k v] results] (println k " " v)) + (do results)))) + +(comment + (do + (set! *unchecked-math* false) + (bench {:all? true})) + + ;; 2023 Aug 1, 2020 Apple MBP M1 + ;; [com.taoensso/nippy "3.2.0"] + ;; [org.clojure/tools.reader "1.3.6"] + ;; [org.clojure/data.fressian "1.0.0"] + + {:reader {:round 35041, :freeze 5942, :thaw 29099, :size 39389} + :fressian {:round 6241, :freeze 3429, :thaw 2812, :size 22850} + :nippy/lzma2 {:round 33561, :freeze 20530, :thaw 13031, :size 11444} + :nippy/encrypted {:round 3390, :freeze 1807, :thaw 1583, :size 16468} + :nippy/default {:round 2845, :freeze 1513, :thaw 1332, :size 16440} + :nippy/fast {:round 2634, :freeze 1338, :thaw 1296, :size 28454}}) diff --git a/test/taoensso/nippy_tests.clj b/test/taoensso/nippy_tests.clj index 644132d..a63b884 100644 --- a/test/taoensso/nippy_tests.clj +++ b/test/taoensso/nippy_tests.clj @@ -393,6 +393,4 @@ ;;;; Benchmarks -(deftest _benchmarks - (is (benchmarks/bench {})) ; Also tests :cached passwords - ) +(deftest _benchmarks (is (benchmarks/bench {})))