diff --git a/CHANGES.md b/CHANGES.md index 420e393..f0cc5fb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,14 +1,17 @@ -## 0.11.1 (unreleased) +## 0.11.1 * More efficient inline caching for Clojure version, benchmarks show inline caching within 5% of manually precompiled code for all cases +* Added navigators for transients in com.rpl.specter.transient namespace (thanks @aaengelberg) * Huge performance improvement for ALL transform on maps and vectors * Significant performance improvements for FIRST/LAST for vectors * Huge performance improvements for `if-path`, `cond-path`, `selected?`, and `not-selected?`, especially for condition path containing only static functions +* Huge performance improvement for `END` on vectors +* Added specialized MAP-VALS navigator that is twice as fast as using [ALL LAST] * Eliminated compiler warnings for ClojureScript version * Dropped support for Clojurescript below v1.7.10 * Added :notpath metadata to signify pathedfn arguments that should be treated as regular arguments during inline factoring. If one of these arguments is not a static var reference or non-collection value, the path will not factor. * Bug fix: `transformed` transform-fn no longer factors into `pred` when an anonymous function during inline factoring * Bug fix: Fixed nil->val to not replace the val on `false` -* Bug fix: Eliminate reflection when using primitive paramaters in an inline cached path +* Bug fix: Eliminate reflection when using primitive parameters in an inline cached path ## 0.11.0 * New `path` macro does intelligent inline caching of the provided path. The path is factored into a static portion and into params which may change on each usage of the path (e.g. local parameters). The static part is factored and compiled on the first run-through, and then re-used for all subsequent invocations. As an example, `[ALL (keypath k)]` is factored into `[ALL keypath]`, which is compiled and cached, and `[k]`, which is provided on each execution. If it is not possible to precompile the path (e.g. [ALL some-local-variable]), nothing is cached and the path will be compiled on each run-through. diff --git a/README.md b/README.md index 1bcf6be..b9a1d8c 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,8 @@ Specter's API is contained in three files: - [macros.clj](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter/macros.clj): This contains the core `select/transform/etc.` operations as well as macros for defining new navigators. - [specter.cljx](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter.cljx): This contains the build-in navigators and functional versions of `select/transform/etc.` -- [zippers.cljx](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter/zipper.cljx): This integrates zipper-based navigation into Specter. +- [transient.cljx](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter/transient.cljx): This contains navigators for transient collections. +- [zipper.cljx](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter/zipper.cljx): This integrates zipper-based navigation into Specter. # Questions? diff --git a/VERSION b/VERSION index 9ba95d1..af88ba8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.11.1-SNAPSHOT +0.11.1 diff --git a/project.clj b/project.clj index 9f2ab74..fd2966b 100644 --- a/project.clj +++ b/project.clj @@ -14,7 +14,9 @@ :codox {:source-paths ["target/classes" "src/clj"] :namespaces [com.rpl.specter com.rpl.specter.macros - com.rpl.specter.zipper] + com.rpl.specter.zipper + com.rpl.specter.protocols + com.rpl.specter.transient] :source-uri {#"target/classes" "https://github.com/nathanmarz/specter/tree/{version}/src/clj/{classpath}x#L{line}" #".*" "https://github.com/nathanmarz/specter/tree/{version}/src/clj/{classpath}#L{line}" diff --git a/scripts/benchmarks.clj b/scripts/benchmarks.clj index 6f3ac43..4396209 100644 --- a/scripts/benchmarks.clj +++ b/scripts/benchmarks.clj @@ -1,6 +1,7 @@ (ns com.rpl.specter.benchmarks (:use [com.rpl.specter] [com.rpl.specter macros] + [com.rpl.specter.transient] [com.rpl.specter.impl :only [benchmark]]) (:require [clojure.walk :as walk])) @@ -30,7 +31,7 @@ (time-ms amt-per-iter afn)))) (defn compare-benchmark [amt-per-iter afn-map] - (let [results (transform [ALL LAST] + (let [results (transform MAP-VALS (fn [afn] (average-time-ms 8 amt-per-iter afn)) afn-map) @@ -49,13 +50,12 @@ (println "\n********************************\n") ))) - - (let [data {:a {:b {:c 1}}} p (comp-paths :a :b :c)] (run-benchmark "get value in nested map" 10000000 (get-in data [:a :b :c]) (select [:a :b :c] data) + (select [(keypath :a) (keypath :b) (keypath :c)] data) (compiled-select p data) (-> data :a :b :c vector) ) @@ -87,6 +87,12 @@ (transform ALL inc data) )) +(let [v (vec (range 1000))] + (run-benchmark "END on large vector" + 5000000 + (setval END [1] v) + (reduce conj v [1]) + (conj v 1))) (defn- update-pair [[k v]] [k (inc v)]) @@ -106,6 +112,7 @@ (reduce-kv (fn [m k v] (assoc m k (inc v))) {} data) (manual-similar-reduce-kv data) (transform [ALL LAST] inc data) + (transform MAP-VALS inc data) )) (let [data (->> (for [i (range 1000)] [i i]) (into {}))] @@ -114,6 +121,7 @@ (reduce-kv (fn [m k v] (assoc m k (inc v))) {} data) (manual-similar-reduce-kv data) (transform [ALL LAST] inc data) + (transform MAP-VALS inc data) )) @@ -147,3 +155,54 @@ (tree-value-transform (fn [e] (if (even? e) (inc e) e)) data) )) +(let [toappend (range 1000)] + (run-benchmark "transient comparison: building up vectors" + 10000 + (reduce (fn [v i] (conj v i)) [] toappend) + (reduce (fn [v i] (conj! v i)) (transient []) toappend) + (setval END toappend []) + (setval END! toappend (transient [])))) + +(let [toappend (range 1000)] + (run-benchmark "transient comparison: building up vectors one at a time" + 10000 + (reduce (fn [v i] (conj v i)) [] toappend) + (reduce (fn [v i] (conj! v i)) (transient []) toappend) + (reduce (fn [v i] (setval END [i] v)) [] toappend) + (reduce (fn [v i] (setval END! [i] v)) (transient []) toappend) + )) + +(let [data (vec (range 1000)) + tdata (transient data) + tdata2 (transient data) + idx 600] + (run-benchmark "transient comparison: assoc'ing in vectors" + 2500000 + (assoc data idx 0) + (assoc! tdata idx 0) + (setval (keypath idx) 0 data) + (setval (keypath! idx) 0 tdata2))) + +(let [data (into {} (for [k (range 1000)] + [k (rand)])) + tdata (transient data) + tdata2 (transient data) + idx 600] + (run-benchmark "transient comparison: assoc'ing in maps" + 2500000 + (assoc data idx 0) + (assoc! tdata idx 0) + (setval (keypath idx) 0 data) + (setval (keypath! idx) 0 tdata2))) + +(defn modify-submap + [m] + (assoc m 0 1 458 89)) + +(let [data (into {} (for [k (range 1000)] + [k (rand)])) + tdata (transient data)] + (run-benchmark "transient comparison: submap" + 300000 + (transform (submap [600 700]) modify-submap data) + (transform (submap! [600 700]) modify-submap tdata))) diff --git a/src/clj/com/rpl/specter.cljx b/src/clj/com/rpl/specter.cljx index 756d127..d55b8e9 100644 --- a/src/clj/com/rpl/specter.cljx +++ b/src/clj/com/rpl/specter.cljx @@ -163,6 +163,18 @@ ALL (comp-paths (i/->AllNavigator))) +(defnav + ^{:doc "Navigate to each value of the map. This is more efficient than + navigating via [ALL LAST]"} + MAP-VALS + [] + (select* [this structure next-fn] + (doall (mapcat next-fn (vals structure)))) + (transform* [this structure next-fn] + (i/map-vals-transform structure next-fn) + )) + + (def VAL (i/->ValCollect)) (def @@ -217,16 +229,28 @@ (reverse (i/matching-ranges structure pred)) ))) -(def +(defnav ^{:doc "Navigate to the empty subsequence before the first element of the collection."} BEGINNING - (srange 0 0)) + [] + (select* [this structure next-fn] + (next-fn [])) + (transform* [this structure next-fn] + (let [to-prepend (next-fn [])] + (i/prepend-all structure to-prepend) + ))) -(def +(defnav ^{:doc "Navigate to the empty subsequence after the last element of the collection."} END - (srange-dynamic count count)) - + [] + (select* [this structure next-fn] + (next-fn [])) + (transform* [this structure next-fn] + (let [to-append (next-fn [])] + (i/append-all structure to-append) + ))) + (defnav ^{:doc "Navigates to the specified subset (by taking an intersection). In a transform, that subset in the original set is changed to the diff --git a/src/clj/com/rpl/specter/impl.cljx b/src/clj/com/rpl/specter/impl.cljx index 196a29d..3f1b3ee 100644 --- a/src/clj/com/rpl/specter/impl.cljx +++ b/src/clj/com/rpl/specter/impl.cljx @@ -448,6 +448,30 @@ (defn- append [coll elem] (-> coll vec (conj elem))) +(defprotocol AddExtremes + (append-all [structure elements]) + (prepend-all [structure elements])) + +(extend-protocol AddExtremes + #+clj clojure.lang.PersistentVector #+cljs cljs.core/PersistentVector + (append-all [structure elements] + (reduce conj structure elements)) + (prepend-all [structure elements] + (let [ret (transient [])] + (as-> ret <> + (reduce conj! <> elements) + (reduce conj! <> structure) + (persistent! <>) + ))) + + #+clj Object #+cljs default + (append-all [structure elements] + (concat structure elements)) + (prepend-all [structure elements] + (concat elements structure)) + ) + + (defprotocol UpdateExtremes (update-first [s afn]) (update-last [s afn])) @@ -473,6 +497,14 @@ (defn vec-count [v] (count v)) +#+clj +(defn transient-vec-count [^clojure.lang.ITransientVector v] + (.count v)) + +#+cljs +(defn transient-vec-count [v] + (count v)) + (extend-protocol UpdateExtremes #+clj clojure.lang.PersistentVector #+cljs cljs.core/PersistentVector (update-first [v afn] @@ -512,6 +544,9 @@ #+clj clojure.lang.IPersistentVector #+cljs cljs.core/PersistentVector (fast-empty? [v] (= 0 (vec-count v))) + #+clj clojure.lang.ITransientVector #+cljs cljs.core/TransientVector + (fast-empty? [v] + (= 0 (transient-vec-count v))) #+clj Object #+cljs default (fast-empty? [s] (empty? s)) @@ -702,6 +737,49 @@ (transform* [this structure next-fn] (all-transform structure next-fn))) +(defprotocol MapValsTransformProtocol + (map-vals-transform [structure next-fn])) + +(defn map-vals-non-transient-transform [structure empty-map next-fn] + (reduce-kv + (fn [m k v] + (assoc m k (next-fn v))) + empty-map + structure)) + +(extend-protocol MapValsTransformProtocol + #+clj clojure.lang.PersistentArrayMap #+cljs cljs.core/PersistentArrayMap + (map-vals-transform [structure next-fn] + (map-vals-non-transient-transform structure {} next-fn) + ) + + #+clj clojure.lang.PersistentTreeMap #+cljs cljs.core/PersistentTreeMap + (map-vals-transform [structure next-fn] + (map-vals-non-transient-transform structure (sorted-map) next-fn) + ) + + #+clj clojure.lang.PersistentHashMap #+cljs cljs.core/PersistentHashMap + (map-vals-transform [structure next-fn] + (persistent! + (reduce-kv + (fn [m k v] + (assoc! m k (next-fn v))) + (transient + #+clj clojure.lang.PersistentHashMap/EMPTY #+cljs cljs.core.PersistentHashMap.EMPTY + ) + structure + ))) + + #+clj Object #+cljs default + (map-vals-transform [structure next-fn] + (reduce-kv + (fn [m k v] + (assoc m k (next-fn v))) + (empty structure) + structure)) + ) + + (deftype ValCollect []) (extend-protocol p/Collector @@ -764,6 +842,16 @@ (next-fn structure) )) +(deftype TransientEndNavigator []) + +(extend-protocol p/Navigator + TransientEndNavigator + (select* [this structure next-fn] + (next-fn [])) + (transform* [this structure next-fn] + (let [res (next-fn [])] + (reduce conj! structure res)))) + (defn extract-basic-filter-fn [path] (cond (fn? path) path diff --git a/src/clj/com/rpl/specter/transient.cljx b/src/clj/com/rpl/specter/transient.cljx new file mode 100644 index 0000000..774e19f --- /dev/null +++ b/src/clj/com/rpl/specter/transient.cljx @@ -0,0 +1,91 @@ +(ns com.rpl.specter.transient + #+cljs + (:require-macros [com.rpl.specter.macros + :refer + [defnav + defpathedfn]]) + (:use #+clj + [com.rpl.specter.macros :only + [defnav + defpathedfn]]) + (:require [com.rpl.specter.impl :as i] + [com.rpl.specter :refer [subselect selected?]])) + +(defnav + ^{:doc "Navigates to the specified key of a transient collection, + navigating to nil if it doesn't exist."} + keypath! + [key] + (select* [this structure next-fn] + (next-fn (get structure key))) + (transform* [this structure next-fn] + (assoc! structure key (next-fn (get structure key))))) + +(def END! + "Navigates to an empty (persistent) vector at the end of a transient vector." + (i/comp-paths* [(i/->TransientEndNavigator)])) + +(defn- t-get-first + [tv] + (nth tv 0)) + +(defn- t-get-last + [tv] + (nth tv (dec (i/transient-vec-count tv)))) + +(defn- t-update-first + [tv next-fn] + (assoc! tv 0 (next-fn (nth tv 0)))) + +(defn- t-update-last + [tv next-fn] + (let [i (dec (i/transient-vec-count tv))] + (assoc! tv i (next-fn (nth tv i))))) + +(def FIRST! + "Navigates to the first element of a transient vector." + (i/->PosNavigator t-get-first t-update-first)) + +(def LAST! + "Navigates to the last element of a transient vector." + (i/->PosNavigator t-get-last t-update-last)) + +#+clj +(defn- select-keys-from-transient-map + "Selects keys from transient map, because built-in select-keys uses + `find` which is unsupported." + [m m-keys] + (loop [result {} + m-keys m-keys] + (if-not (seq m-keys) + result + (let [k (first m-keys) + ;; support Clojure 1.6 where contains? is broken on transients + item (get m k ::not-found)] + (recur (if-not (identical? item ::not-found) + (assoc result k item) + result) + (rest m-keys)))))) + +#+cljs +(defn- select-keys-from-transient-map + "Uses select-keys on a transient map." + [m m-keys] + (select-keys m m-keys)) + +(defnav + ^{:doc "Navigates to the specified persistent submap of a transient map."} + submap! + [m-keys] + (select* [this structure next-fn] + (next-fn (select-keys-from-transient-map structure m-keys))) + (transform* [this structure next-fn] + (let [selected (select-keys-from-transient-map structure m-keys) + res (next-fn selected)] + (as-> structure % + (reduce (fn [m k] + (dissoc! m k)) + % m-keys) + (reduce-kv (fn [m k v] + (assoc! m k v)) + % res))))) diff --git a/test/com/rpl/specter/core_test.cljx b/test/com/rpl/specter/core_test.cljx index cf0fbb1..9aa1510 100644 --- a/test/com/rpl/specter/core_test.cljx +++ b/test/com/rpl/specter/core_test.cljx @@ -25,6 +25,7 @@ #+cljs [cljs.test.check.generators :as gen] #+cljs [cljs.test.check.properties :as prop :include-macros true] [com.rpl.specter :as s] + [com.rpl.specter.transient :as t] [clojure.set :as set])) ;;TODO: @@ -64,8 +65,9 @@ (defspec select-all-on-map (for-all+ - [m (limit-size 5 (gen/map gen/keyword gen/int))] - (= (select [s/ALL s/LAST] m) + [m (limit-size 5 (gen/map gen/keyword gen/int)) + p (gen/elements [s/MAP-VALS [s/ALL s/LAST]])] + (= (select p m) (for [[k v] m] v)) )) @@ -81,8 +83,9 @@ (defspec transform-all-on-map (for-all+ - [m (limit-size 5 (gen/map gen/keyword gen/int))] - (= (transform [s/ALL s/LAST] inc m) + [m (limit-size 5 (gen/map gen/keyword gen/int)) + p (gen/elements [s/MAP-VALS [s/ALL s/LAST]])] + (= (transform p inc m) (into {} (for [[k v] m] [k (inc v)])) ))) @@ -1036,3 +1039,34 @@ (is (= 2 (key e))) (is (= 4 (val e))) )) + +(defspec transient-vector-test + (for-all+ + [v (gen/vector (limit-size 5 gen/int))] + (every? identity + (for [[path transient-path f] + [[s/FIRST t/FIRST! (fnil inc 0)] ;; fnil in case vector is empty + [s/LAST t/LAST! (fnil inc 0)] + [(s/keypath 0) (t/keypath! 0) (fnil inc 0)] + [s/END t/END! #(conj % 1 2 3)]]] + (and (= (s/transform* path f v) + (persistent! (s/transform* transient-path f (transient v)))) + (= (s/select* path v) + (s/select* transient-path (transient v)))))))) + +(defspec transient-map-test + (for-all+ + [m (limit-size 5 (gen/not-empty (gen/map gen/keyword gen/int))) + new-key gen/keyword] + (let [existing-key (first (keys m))] + (every? identity + (for [[path transient-path f] + [[(s/keypath existing-key) (t/keypath! existing-key) inc] + [(s/keypath new-key) (t/keypath! new-key) (constantly 3)] + [(s/submap [existing-key new-key]) + (t/submap! [existing-key new-key]) + (constantly {new-key 1234})]]] + (and (= (s/transform* path f m) + (persistent! (s/transform* transient-path f (transient m)))) + (= (s/select* path m) + (s/select* transient-path (transient m)))))))))