luciano/specter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #275 from xificurC/master

Browse source 
use criterium for benchmarks
This commit is contained in:
Nathan Marz 2018-12-13 21:23:03 -05:00 committed by GitHub
commit e38a2561d8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 36 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
project.clj
View file

@ -32,7 +32,8 @@
[[org.clojure/test.check "0.9.0"] [[org.clojure/test.check "0.9.0"]
[org.clojure/clojure "1.9.0"] [org.clojure/clojure "1.9.0"]
[org.clojure/clojurescript "1.10.439"]]} [org.clojure/clojurescript "1.10.439"]]}
:bench {:dependencies [[org.clojure/clojure "1.9.0"]
[criterium "0.4.4"]]}
:test {:dependencies [[org.clojure/clojure "1.7.0"]]}} :test {:dependencies [[org.clojure/clojure "1.7.0"]]}}
:aliases {"deploy" ["do" "clean," "deploy" "clojars"]}) :aliases {"deploy" ["do" "clean," "deploy" "clojars"]})

95
scripts/benchmarks.clj
View file

@ -2,56 +2,27 @@
(:use [com.rpl.specter] (:use [com.rpl.specter]
[com.rpl.specter.transients]) [com.rpl.specter.transients])
(:require [clojure.walk :as walk] (:require [clojure.walk :as walk]
[com.rpl.specter.impl :as i])) [com.rpl.specter.impl :as i]
[criterium.core :as bench]))
;; run via `lein repl` with `(load-file "scripts/benchmarks.clj")`
(defn pretty-float5 [anum]
(format "%.5g" anum))
(defn pretty-float3 [anum] (defn pretty-float3 [anum]
(format "%.3g" anum)) (format "%.3g" anum))
(defn time-ms [amt afn] (defn mean [a-fn]
(let [start (System/nanoTime) (-> a-fn (bench/benchmark* {}) :mean first (* 1000000)))
_ (dotimes [_ amt] (afn))
end (System/nanoTime)]
(/ (- end start) 1000000.0)))
(defn compare-benchmark [afn-map]
(defn avg [numbers] (let [results (transform MAP-VALS mean afn-map)
(/ (reduce + numbers)
(count numbers)
1.0))
(defn average-time-ms [iters amt-per-iter afn]
(avg
;; treat 1st run as warmup
(next
(for [i (range (inc iters))]
(time-ms amt-per-iter afn)))))
(defn compare-benchmark [amt-per-iter afn-map]
(System/runFinalization)
(System/gc)
(let [results (transform MAP-VALS
(fn [afn]
(average-time-ms 8 amt-per-iter afn))
afn-map)
[[_ best-time] & _ :as sorted] (sort-by last results)] [[_ best-time] & _ :as sorted] (sort-by last results)]
(println "\nMean(us)\tvs best\t\tCode")
(println "\nAvg(ms)\t\tvs best\t\tCode")
(doseq [[k t] sorted] (doseq [[k t] sorted]
(println (pretty-float5 t) "\t\t" (pretty-float3 (/ t best-time 1.0)) "\t\t" k)))) (println (pretty-float3 t) "\t\t" (pretty-float3 (/ t best-time 1.0)) "\t\t" k))))
(defmacro run-benchmark [name & exprs]
(defmacro run-benchmark [name amt-per-iter & exprs]
(let [afn-map (->> exprs shuffle (map (fn [e] [`(quote ~e) `(fn [] ~e)])) (into {}))] (let [afn-map (->> exprs shuffle (map (fn [e] [`(quote ~e) `(fn [] ~e)])) (into {}))]
`(do `(do
(println "Benchmark:" ~name (str "(" ~amt-per-iter " iterations)")) (println "Benchmark:" ~name)
(compare-benchmark ~amt-per-iter ~afn-map) (compare-benchmark ~afn-map)
(println "\n********************************\n")))) (println "\n********************************\n"))))
(defn specter-dynamic-nested-get [data a b c] (defn specter-dynamic-nested-get [data a b c]
@ -69,7 +40,7 @@
(let [data {:a {:b {:c 1}}} (let [data {:a {:b {:c 1}}}
p (comp-paths :a :b :c)] p (comp-paths :a :b :c)]
(run-benchmark "get value in nested map" 2500000 (run-benchmark "get value in nested map"
(select-any [:a :b :c] data) (select-any [:a :b :c] data)
(select-any (keypath :a :b :c) data) (select-any (keypath :a :b :c) data)
(select-one [:a :b :c] data) (select-one [:a :b :c] data)
@ -86,7 +57,7 @@
(let [data {:a {:b {:c 1}}}] (let [data {:a {:b {:c 1}}}]
(run-benchmark "set value in nested map" 2500000 (run-benchmark "set value in nested map"
(assoc-in data [:a :b :c] 1) (assoc-in data [:a :b :c] 1)
(setval [:a :b :c] 1 data))) (setval [:a :b :c] 1 data)))
@ -103,7 +74,7 @@
(my-update m3 :c afn)))))) (my-update m3 :c afn))))))
(let [data {:a {:b {:c 1}}}] (let [data {:a {:b {:c 1}}}]
(run-benchmark "update value in nested map" 500000 (run-benchmark "update value in nested map"
(update-in data [:a :b :c] inc) (update-in data [:a :b :c] inc)
(transform [:a :b :c] inc data) (transform [:a :b :c] inc data)
(manual-transform data inc))) (manual-transform data inc)))
@ -135,14 +106,14 @@
(let [data '(1 2 3 4 5)] (let [data '(1 2 3 4 5)]
(run-benchmark "transform values of a list" 500000 (run-benchmark "transform values of a list"
(transform ALL inc data) (transform ALL inc data)
(doall (sequence (map inc) data)) (doall (sequence (map inc) data))
(reverse (into '() (map inc) data)) (reverse (into '() (map inc) data))
)) ))
(let [data {:a 1 :b 2 :c 3 :d 4}] (let [data {:a 1 :b 2 :c 3 :d 4}]
(run-benchmark "transform values of a small map" 500000 (run-benchmark "transform values of a small map"
(into {} (for [[k v] data] [k (inc v)])) (into {} (for [[k v] data] [k (inc v)]))
(reduce-kv (fn [m k v] (assoc m k (inc v))) {} data) (reduce-kv (fn [m k v] (assoc m k (inc v))) {} data)
(persistent! (reduce-kv (fn [m k v] (assoc! m k (inc v))) (transient {}) data)) (persistent! (reduce-kv (fn [m k v] (assoc! m k (inc v))) (transient {}) data))
@ -158,7 +129,7 @@
(let [data (->> (for [i (range 1000)] [i i]) (into {}))] (let [data (->> (for [i (range 1000)] [i i]) (into {}))]
(run-benchmark "transform values of large map" 600 (run-benchmark "transform values of large map"
(into {} (for [[k v] data] [k (inc v)])) (into {} (for [[k v] data] [k (inc v)]))
(reduce-kv (fn [m k v] (assoc m k (inc v))) {} data) (reduce-kv (fn [m k v] (assoc m k (inc v))) {} data)
(persistent! (reduce-kv (fn [m k v] (assoc! m k (inc v))) (transient {}) data)) (persistent! (reduce-kv (fn [m k v] (assoc! m k (inc v))) (transient {}) data))
@ -174,7 +145,7 @@
(let [data [1 2 3 4 5 6 7 8 9 10]] (let [data [1 2 3 4 5 6 7 8 9 10]]
(run-benchmark "first value of a size 10 vector" 10000000 (run-benchmark "first value of a size 10 vector"
(first data) (first data)
(select-any ALL data) (select-any ALL data)
(select-any FIRST data) (select-any FIRST data)
@ -182,7 +153,7 @@
)) ))
(let [data [1 2 3 4 5]] (let [data [1 2 3 4 5]]
(run-benchmark "map a function over a vector" 1000000 (run-benchmark "map a function over a vector"
(vec (map inc data)) (vec (map inc data))
(mapv inc data) (mapv inc data)
(transform ALL inc data) (transform ALL inc data)
@ -190,7 +161,7 @@
(let [data [1 2 3 4 5 6 7 8 9 10]] (let [data [1 2 3 4 5 6 7 8 9 10]]
(run-benchmark "filter a sequence" 500000 (run-benchmark "filter a sequence"
(doall (filter even? data)) (doall (filter even? data))
(filterv even? data) (filterv even? data)
(select [ALL even?] data) (select [ALL even?] data)
@ -200,7 +171,7 @@
(let [data [{:a 2 :b 2} {:a 1} {:a 4} {:a 6}] (let [data [{:a 2 :b 2} {:a 1} {:a 4} {:a 6}]
xf (comp (map :a) (filter even?))] xf (comp (map :a) (filter even?))]
(run-benchmark "even :a values from sequence of maps" 500000 (run-benchmark "even :a values from sequence of maps"
(select [ALL :a even?] data) (select [ALL :a even?] data)
(->> data (mapv :a) (filter even?) doall) (->> data (mapv :a) (filter even?) doall)
(into [] (comp (map :a) (filter even?)) data) (into [] (comp (map :a) (filter even?)) data)
@ -209,14 +180,13 @@
(let [v (vec (range 1000))] (let [v (vec (range 1000))]
(run-benchmark "Append to a large vector" (run-benchmark "Append to a large vector"
2000000
(setval END [1] v) (setval END [1] v)
(setval AFTER-ELEM 1 v) (setval AFTER-ELEM 1 v)
(reduce conj v [1]) (reduce conj v [1])
(conj v 1))) (conj v 1)))
(let [data [1 2 3 4 5 6 7 8 9 10]] (let [data [1 2 3 4 5 6 7 8 9 10]]
(run-benchmark "prepend to a vector" 1000000 (run-benchmark "prepend to a vector"
(vec (cons 0 data)) (vec (cons 0 data))
(setval BEFORE-ELEM 0 data) (setval BEFORE-ELEM 0 data)
(into [0] data) (into [0] data)
@ -244,7 +214,6 @@
(let [data [1 2 [[3]] [4 6 [7 [8]] 10]]] (let [data [1 2 [[3]] [4 6 [7 [8]] 10]]]
(run-benchmark "update every value in a tree (represented with vectors)" (run-benchmark "update every value in a tree (represented with vectors)"
50000
(walk/postwalk (fn [e] (if (and (number? e) (even? e)) (inc e) e)) data) (walk/postwalk (fn [e] (if (and (number? e) (even? e)) (inc e) e)) data)
(transform [(walker number?) even?] inc data) (transform [(walker number?) even?] inc data)
(transform [TreeValues even?] inc data) (transform [TreeValues even?] inc data)
@ -254,7 +223,6 @@
(let [toappend (range 1000)] (let [toappend (range 1000)]
(run-benchmark "transient comparison: building up vectors" (run-benchmark "transient comparison: building up vectors"
8000
(reduce (fn [v i] (conj v i)) [] toappend) (reduce (fn [v i] (conj v i)) [] toappend)
(reduce (fn [v i] (conj! v i)) (transient []) toappend) (reduce (fn [v i] (conj! v i)) (transient []) toappend)
(setval END toappend []) (setval END toappend [])
@ -262,7 +230,6 @@
(let [toappend (range 1000)] (let [toappend (range 1000)]
(run-benchmark "transient comparison: building up vectors one at a time" (run-benchmark "transient comparison: building up vectors one at a time"
7000
(reduce (fn [v i] (conj v i)) [] toappend) (reduce (fn [v i] (conj v i)) [] toappend)
(reduce (fn [v i] (conj! v i)) (transient []) toappend) (reduce (fn [v i] (conj! v i)) (transient []) toappend)
(reduce (fn [v i] (setval END [i] v)) [] toappend) (reduce (fn [v i] (setval END [i] v)) [] toappend)
@ -273,7 +240,6 @@
tdata (transient data) tdata (transient data)
tdata2 (transient data)] tdata2 (transient data)]
(run-benchmark "transient comparison: assoc'ing in vectors" (run-benchmark "transient comparison: assoc'ing in vectors"
2500000
(assoc data 600 0) (assoc data 600 0)
(assoc! tdata 600 0) (assoc! tdata 600 0)
(setval (keypath 600) 0 data) (setval (keypath 600) 0 data)
@ -284,7 +250,6 @@
tdata (transient data) tdata (transient data)
tdata2 (transient data)] tdata2 (transient data)]
(run-benchmark "transient comparison: assoc'ing in maps" (run-benchmark "transient comparison: assoc'ing in maps"
1500000
(assoc data 600 0) (assoc data 600 0)
(assoc! tdata 600 0) (assoc! tdata 600 0)
(setval (keypath 600) 0 data) (setval (keypath 600) 0 data)
@ -298,31 +263,27 @@
[k (rand)])) [k (rand)]))
tdata (transient data)] tdata (transient data)]
(run-benchmark "transient comparison: submap" (run-benchmark "transient comparison: submap"
150000
(transform (submap [600 700]) modify-submap data) (transform (submap [600 700]) modify-submap data)
(transform (submap! [600 700]) modify-submap tdata))) (transform (submap! [600 700]) modify-submap tdata)))
(let [data {:x 1} (let [data {:x 1}
meta-map {:my :metadata}] meta-map {:my :metadata}]
(run-benchmark "set metadata" (run-benchmark "set metadata"
1500000
(with-meta data meta-map) (with-meta data meta-map)
(setval META meta-map data))) (setval META meta-map data)))
(let [data (with-meta {:x 1} {:my :metadata})] (let [data (with-meta {:x 1} {:my :metadata})]
(run-benchmark "get metadata" (run-benchmark "get metadata"
15000000
(meta data) (meta data)
(select-any META data))) (select-any META data)))
(let [data (with-meta {:x 1} {:my :metadata})] (let [data (with-meta {:x 1} {:my :metadata})]
(run-benchmark "vary metadata" (run-benchmark "vary metadata"
800000
(vary-meta data assoc :y 2) (vary-meta data assoc :y 2)
(setval [META :y] 2 data))) (setval [META :y] 2 data)))
(let [data (range 1000)] (let [data (range 1000)]
(run-benchmark "Traverse into a set" 5000 (run-benchmark "Traverse into a set"
(set data) (set data)
(set (select ALL data)) (set (select ALL data))
(into #{} (traverse ALL data)) (into #{} (traverse ALL data))
@ -334,18 +295,18 @@
(defn mult-10 [v] (* 10 v)) (defn mult-10 [v] (* 10 v))
(let [data [1 2 3 4 5 6 7 8 9]] (let [data [1 2 3 4 5 6 7 8 9]]
(run-benchmark "multi-transform vs. consecutive transforms, one shared nav" 300000 (run-benchmark "multi-transform vs. consecutive transforms, one shared nav"
(->> data (transform [ALL even?] mult-10) (transform [ALL odd?] dec)) (->> data (transform [ALL even?] mult-10) (transform [ALL odd?] dec))
(multi-transform [ALL (multi-path [even? (terminal mult-10)] [odd? (terminal dec)])] data))) (multi-transform [ALL (multi-path [even? (terminal mult-10)] [odd? (terminal dec)])] data)))
(let [data [[1 2 3 4 :a] [5] [6 7 :b 8 9] [10 11 12 13]]] (let [data [[1 2 3 4 :a] [5] [6 7 :b 8 9] [10 11 12 13]]]
(run-benchmark "multi-transform vs. consecutive transforms, three shared navs" 150000 (run-benchmark "multi-transform vs. consecutive transforms, three shared navs"
(->> data (transform [ALL ALL number? even?] mult-10) (transform [ALL ALL number? odd?] dec)) (->> data (transform [ALL ALL number? even?] mult-10) (transform [ALL ALL number? odd?] dec))
(multi-transform [ALL ALL number? (multi-path [even? (terminal mult-10)] [odd? (terminal dec)])] data))) (multi-transform [ALL ALL number? (multi-path [even? (terminal mult-10)] [odd? (terminal dec)])] data)))
(let [data {:a 1 :b 2 :c 3 :d 4}] (let [data {:a 1 :b 2 :c 3 :d 4}]
(run-benchmark "namespace qualify keys of a small map" 1000000 (run-benchmark "namespace qualify keys of a small map"
(into {} (into {}
(map (fn [[k v]] [(keyword (str *ns*) (name k)) v])) (map (fn [[k v]] [(keyword (str *ns*) (name k)) v]))
data) data)
@ -355,7 +316,7 @@
(let [data (->> (for [i (range 1000)] [(keyword (str i)) i]) (into {}))] (let [data (->> (for [i (range 1000)] [(keyword (str i)) i]) (into {}))]
(run-benchmark "namespace qualify keys of a large map" 1200 (run-benchmark "namespace qualify keys of a large map"
(into {} (into {}
(map (fn [[k v]] [(keyword (str *ns*) (name k)) v])) (map (fn [[k v]] [(keyword (str *ns*) (name k)) v]))
data) data)
@ -370,7 +331,7 @@
(i/walk-until afn next-fn structure))) (i/walk-until afn next-fn structure)))
(let [data {:a [1 2 {:c '(3 4) :d {:e [1 2 3] 7 8 9 10}}]}] (let [data {:a [1 2 {:c '(3 4) :d {:e [1 2 3] 7 8 9 10}}]}]
(run-benchmark "walker vs. clojure.walk version" 150000 (run-benchmark "walker vs. clojure.walk version"
(transform (walker number?) inc data) (transform (walker number?) inc data)
(transform (walker-old number?) inc data) (transform (walker-old number?) inc data)
)) ))

8
scripts/run-benchmarks
View file

@ -1,4 +1,8 @@
#!/bin/bash #!/bin/bash
`lein javac` lein javac
java -server -XX:MaxPermSize=128m -XX:MaxInlineSize=100 -cp `lein classpath` clojure.main scripts/benchmarks.clj lein version
echo
lein show-profiles bench
echo
java -server -XX:MaxInlineSize=100 -cp "$(lein with-profile bench classpath)" clojure.main scripts/benchmarks.clj