From 1dd6a38f050acf9fe1ae24f9741a322c3a5e8fc9 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 30 Sep 2020 10:25:15 -0600 Subject: [PATCH] Used minimal encore and sped up requiring by 2/3 time --- project.clj | 2 +- src/taoensso/min_encore.clj | 48 ++++++++++++++++++++++++++++-- src/taoensso/nippy.clj | 11 ++++--- src/taoensso/nippy/benchmarks.clj | 2 +- test/taoensso/nippy/tests/main.clj | 10 ++++++- 5 files changed, 61 insertions(+), 12 deletions(-) diff --git a/project.clj b/project.clj index 80a8fde..f83523b 100644 --- a/project.clj +++ b/project.clj @@ -14,7 +14,7 @@ :dependencies [[org.clojure/tools.reader "1.3.3"] - [com.taoensso/encore "3.1.0"] + [com.taoensso/truss "1.6.0"] [org.iq80.snappy/snappy "0.4"] [org.tukaani/xz "1.8"] [org.lz4/lz4-java "1.7.1"]] diff --git a/src/taoensso/min_encore.clj b/src/taoensso/min_encore.clj index 874e838..286c325 100644 --- a/src/taoensso/min_encore.clj +++ b/src/taoensso/min_encore.clj @@ -294,7 +294,7 @@ :else (.computeIfAbsent - cache_ nil-sentinel + cache_ xs (reify Function (apply [this k] (apply f xs)))))))))) @@ -364,7 +364,13 @@ (defmacro thread-local-proxy [& body] `(proxy [ThreadLocal] [] (initialValue [] (do ~@body)))) -(def ^:private srng* (thread-local-proxy (java.security.SecureRandom/getInstanceStrong))) +(def ^:private srng* (thread-local-proxy + (or + ;; Very strong and blocking. See + ;; https://stackoverflow.com/questions/137212/how-to-deal-with-a-slow-securerandom-generator + (java.security.SecureRandom/getInstanceStrong) + ;;Weaker and much faster + #_(java.security.SecureRandom/getInstance "SHA1PRNG")))) (defn secure-rng "Returns a thread-local `java.security.SecureRandom`. @@ -571,4 +577,40 @@ :else (throw (ex-info "compile-str-filter: `allow-spec` and `deny-spec` cannot both be nil" - {:allow-spec allow-spec :deny-spec deny-spec}))))))) + {:allow-spec allow-spec :deny-spec deny-spec}))))))) + + +(defn round0 ^long [n] (Math/round (double n))) +(defn round1 ^double [n] (/ (double (Math/round (* (double n) 10.0))) 10.0)) +(defn round2 ^double [n] (/ (double (Math/round (* (double n) 100.0))) 100.0)) +(defn perc ^long [n divisor] (Math/round (* (/ (double n) (double divisor)) 100.0))) + + +(defmacro now-nano* [] `(System/nanoTime)) + + +(defmacro time-ns "Returns number of nanoseconds it took to execute body." + [& body] `(let [t0# (now-nano*)] ~@body (- (now-nano*) t0#))) + + +(defn bench* + "Repeatedly executes fn and returns time taken to complete execution." + [nlaps {:keys [nlaps-warmup nthreads as-ns?] + :or {nlaps-warmup 0 + nthreads 1}} f] + (try + (dotimes [_ nlaps-warmup] (f)) + (let [nanosecs + (if (= nthreads 1) + (time-ns (dotimes [_ nlaps] (f))) + (let [nlaps-per-thread (/ nlaps nthreads)] + (time-ns + (let [futures (repeatedly-into [] nthreads + (fn [] (future (dotimes [_ nlaps-per-thread] (f)))))] + (mapv deref futures)))))] + (if as-ns? nanosecs (round0 (/ nanosecs 1e6)))) + (catch Throwable t + (println (str "Bench failure: " (.getMessage t))) + -1))) + +(defmacro bench [nlaps opts & body] `(bench* ~nlaps ~opts (fn [] ~@body))) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 45edaf5..2d9d545 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -1325,15 +1325,14 @@ ba (if compressor (compress compressor ba) ba) ba (if encryptor (encrypt encryptor password ba) ba)] - (if no-header? ba (wrap-header ba {:compressor-id (when-let [c compressor] (or (compression/standard-header-ids - (compression/header-id c)) - :else)) + (compression/header-id c)) + :else)) :encryptor-id (when-let [e encryptor] @@ -1794,7 +1793,6 @@ ba (if encryptor (decrypt encryptor password ba) ba) ba (if compressor (decompress compressor ba) ba) dis (DataInputStream. (ByteArrayInputStream. ba))] - (with-cache (thaw-from-in! dis))) (catch Exception e (ex-fn e))))) @@ -1992,10 +1990,11 @@ :stress-record (StressRecord. "data") :stress-type (StressType. "data") - ;; Serializable + ;; ;; Serializable :throwable (Throwable. "Yolo") :exception (try (/ 1 0) (catch Exception e e)) - :ex-info (ex-info "ExInfo" {:data "data"})}) + :ex-info (ex-info "ExInfo" {:data "data"}) + }) (def stress-data-comparable "Reference data with stuff removed that breaks roundtrip equality" diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 0d16a98..32a31a8 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,6 +1,6 @@ (ns taoensso.nippy.benchmarks (:require [clojure.data.fressian :as fressian] - [taoensso.encore :as enc] + [taoensso.min-encore :as enc] [taoensso.nippy :as nippy :refer [freeze thaw]])) (def data #_22 nippy/stress-data-benchable) diff --git a/test/taoensso/nippy/tests/main.clj b/test/taoensso/nippy/tests/main.clj index 7ac88ba..262a73f 100644 --- a/test/taoensso/nippy/tests/main.clj +++ b/test/taoensso/nippy/tests/main.clj @@ -4,10 +4,18 @@ [clojure.test.check :as tc] [clojure.test.check.generators :as tc-gens] [clojure.test.check.properties :as tc-props] - [taoensso.encore :as enc :refer ()] [taoensso.nippy :as nippy :refer (freeze thaw)] [taoensso.nippy.benchmarks :as benchmarks])) +(comment + (78 80 89 0 0 0 67 -122 -15 20) + (def ignored (thaw ba)) + 16302 nil nil + + (def ba (freeze stress-data-comparable)) + (78 80 89 8 0 0 67 -122 -15 20) + ) + (comment (test/run-tests)) (def test-data nippy/stress-data-comparable)