Merge branch 'master' into new-readme

2016-06-08 09:44:12 -04:00 · 2016-06-08 09:44:12 -04:00 · ab38b2db62
commit ab38b2db62
parent 1efb3df8de 3d84d288d1
9 changed files with 319 additions and 17 deletions
--- a/CHANGES.md
+++ b/CHANGES.md
@ -1,14 +1,17 @@
-## 0.11.1 (unreleased)
+## 0.11.1
 * More efficient inline caching for Clojure version, benchmarks show inline caching within 5% of manually precompiled code for all cases
+* Added navigators for transients in com.rpl.specter.transient namespace (thanks @aaengelberg)
 * Huge performance improvement for ALL transform on maps and vectors
 * Significant performance improvements for FIRST/LAST for vectors
 * Huge performance improvements for `if-path`, `cond-path`, `selected?`, and `not-selected?`, especially for condition path containing only static functions
+* Huge performance improvement for `END` on vectors
+* Added specialized MAP-VALS navigator that is twice as fast as using [ALL LAST]
 * Eliminated compiler warnings for ClojureScript version
 * Dropped support for Clojurescript below v1.7.10
 * Added :notpath metadata to signify pathedfn arguments that should be treated as regular arguments during inline factoring. If one of these arguments is not a static var reference or non-collection value, the path will not factor. 
 * Bug fix: `transformed` transform-fn no longer factors into `pred` when an anonymous function during inline factoring
 * Bug fix: Fixed nil->val to not replace the val on `false`
-* Bug fix: Eliminate reflection when using primitive paramaters in an inline cached path
+* Bug fix: Eliminate reflection when using primitive parameters in an inline cached path

 ## 0.11.0
 * New `path` macro does intelligent inline caching of the provided path. The path is factored into a static portion and into params which may change on each usage of the path (e.g. local parameters). The static part is factored and compiled on the first run-through, and then re-used for all subsequent invocations. As an example, `[ALL (keypath k)]` is factored into `[ALL keypath]`, which is compiled and cached, and `[k]`, which is provided on each execution. If it is not possible to precompile the path (e.g. [ALL some-local-variable]), nothing is cached and the path will be compiled on each run-through.
--- a/README.md
+++ b/README.md
@ -115,7 +115,8 @@ Specter's API is contained in three files:

 - [macros.clj](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter/macros.clj): This contains the core `select/transform/etc.` operations as well as macros for defining new navigators.
 - [specter.cljx](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter.cljx): This contains the build-in navigators and functional versions of `select/transform/etc.`
- [zippers.cljx](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter/zipper.cljx): This integrates zipper-based navigation into Specter.
+- [transient.cljx](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter/transient.cljx): This contains navigators for transient collections.
+- [zipper.cljx](https://github.com/nathanmarz/specter/blob/master/src/clj/com/rpl/specter/zipper.cljx): This integrates zipper-based navigation into Specter.

 # Questions?

--- a/2
+++ b/2
@ -1 +1 @@
-0.11.1-SNAPSHOT
+0.11.1
--- a/project.clj
+++ b/project.clj
@ -14,7 +14,9 @@
  :codox {:source-paths ["target/classes" "src/clj"]
          :namespaces [com.rpl.specter
                       com.rpl.specter.macros
-                       com.rpl.specter.zipper]
+                       com.rpl.specter.zipper
+                       com.rpl.specter.protocols
+                       com.rpl.specter.transient]
          :source-uri
            {#"target/classes" "https://github.com/nathanmarz/specter/tree/{version}/src/clj/{classpath}x#L{line}"
             #".*"             "https://github.com/nathanmarz/specter/tree/{version}/src/clj/{classpath}#L{line}"
--- a/scripts/benchmarks.clj
+++ b/scripts/benchmarks.clj
@ -1,6 +1,7 @@
 (ns com.rpl.specter.benchmarks
  (:use [com.rpl.specter]
        [com.rpl.specter macros]
+        [com.rpl.specter.transient]
        [com.rpl.specter.impl :only [benchmark]])
  (:require [clojure.walk :as walk]))

@ -30,7 +31,7 @@
      (time-ms amt-per-iter afn))))

 (defn compare-benchmark [amt-per-iter afn-map]
-  (let [results (transform [ALL LAST]
+  (let [results (transform MAP-VALS
                  (fn [afn]
                    (average-time-ms 8 amt-per-iter afn))
                  afn-map)
@ -49,13 +50,12 @@
       (println "\n********************************\n")
       )))

-
-
 (let [data {:a {:b {:c 1}}}
      p (comp-paths :a :b :c)]
  (run-benchmark "get value in nested map" 10000000
    (get-in data [:a :b :c])
    (select [:a :b :c] data)
+    (select [(keypath :a) (keypath :b) (keypath :c)] data)
    (compiled-select p data)
    (-> data :a :b :c vector)
    )
@ -87,6 +87,12 @@
    (transform ALL inc data)
    ))

+(let [v (vec (range 1000))]
+  (run-benchmark "END on large vector"
+    5000000
+    (setval END [1] v)
+    (reduce conj v [1])
+    (conj v 1)))

 (defn- update-pair [[k v]]
  [k (inc v)])
@ -106,6 +112,7 @@
    (reduce-kv (fn [m k v] (assoc m k (inc v))) {} data)
    (manual-similar-reduce-kv data)
    (transform [ALL LAST] inc data)
+    (transform MAP-VALS inc data)
    ))

 (let [data (->> (for [i (range 1000)] [i i]) (into {}))]
@ -114,6 +121,7 @@
    (reduce-kv (fn [m k v] (assoc m k (inc v))) {} data)
    (manual-similar-reduce-kv data)
    (transform [ALL LAST] inc data)
+    (transform MAP-VALS inc data)
    ))


@ -147,3 +155,54 @@
    (tree-value-transform (fn [e] (if (even? e) (inc e) e)) data)
    ))

+(let [toappend (range 1000)]
+  (run-benchmark "transient comparison: building up vectors"
+    10000
+    (reduce (fn [v i] (conj v i)) [] toappend)
+    (reduce (fn [v i] (conj! v i)) (transient []) toappend)
+    (setval END toappend [])
+    (setval END! toappend (transient []))))
+
+(let [toappend (range 1000)]
+  (run-benchmark "transient comparison: building up vectors one at a time"
+    10000
+    (reduce (fn [v i] (conj v i)) [] toappend)
+    (reduce (fn [v i] (conj! v i)) (transient []) toappend)
+    (reduce (fn [v i] (setval END [i] v)) [] toappend)
+    (reduce (fn [v i] (setval END! [i] v)) (transient []) toappend)
+    ))
+
+(let [data (vec (range 1000))
+      tdata (transient data)
+      tdata2 (transient data)
+      idx 600]
+  (run-benchmark "transient comparison: assoc'ing in vectors"
+    2500000
+    (assoc data idx 0)
+    (assoc! tdata idx 0)
+    (setval (keypath idx) 0 data)
+    (setval (keypath! idx) 0 tdata2)))
+
+(let [data (into {} (for [k (range 1000)]
+                      [k (rand)]))
+      tdata (transient data)
+      tdata2 (transient data)
+      idx 600]
+  (run-benchmark "transient comparison: assoc'ing in maps"
+    2500000
+    (assoc data idx 0)
+    (assoc! tdata idx 0)
+    (setval (keypath idx) 0 data)
+    (setval (keypath! idx) 0 tdata2)))
+
+(defn modify-submap
+  [m]
+  (assoc m 0 1 458 89))
+
+(let [data (into {} (for [k (range 1000)]
+                      [k (rand)]))
+      tdata (transient data)]
+  (run-benchmark "transient comparison: submap"
+    300000
+    (transform (submap [600 700]) modify-submap data)
+    (transform (submap! [600 700]) modify-submap tdata)))
--- a/src/clj/com/rpl/specter.cljx
+++ b/src/clj/com/rpl/specter.cljx
@ -163,6 +163,18 @@
  ALL
  (comp-paths (i/->AllNavigator)))

+(defnav
+  ^{:doc "Navigate to each value of the map. This is more efficient than 
+          navigating via [ALL LAST]"}
+  MAP-VALS
+  []
+  (select* [this structure next-fn]
+    (doall (mapcat next-fn (vals structure))))
+  (transform* [this structure next-fn]
+    (i/map-vals-transform structure next-fn)
+    ))
+
+
 (def VAL (i/->ValCollect))

 (def
@ -217,16 +229,28 @@
      (reverse (i/matching-ranges structure pred))
      )))

-(def
+(defnav
  ^{:doc "Navigate to the empty subsequence before the first element of the collection."}
  BEGINNING
-  (srange 0 0))
+  []
+  (select* [this structure next-fn]
+    (next-fn []))
+  (transform* [this structure next-fn]
+    (let [to-prepend (next-fn [])]
+      (i/prepend-all structure to-prepend)
+      )))

-(def
+(defnav
 ^{:doc "Navigate to the empty subsequence after the last element of the collection."}
  END
-  (srange-dynamic count count))
-
+  []
+  (select* [this structure next-fn]
+    (next-fn []))
+  (transform* [this structure next-fn]
+    (let [to-append (next-fn [])]
+      (i/append-all structure to-append)
+      )))
+  
 (defnav
  ^{:doc "Navigates to the specified subset (by taking an intersection).
          In a transform, that subset in the original set is changed to the
--- a/src/clj/com/rpl/specter/impl.cljx
+++ b/src/clj/com/rpl/specter/impl.cljx
@ -448,6 +448,30 @@
 (defn- append [coll elem]
  (-> coll vec (conj elem)))

+(defprotocol AddExtremes
+  (append-all [structure elements])
+  (prepend-all [structure elements]))
+
+(extend-protocol AddExtremes
+  #+clj clojure.lang.PersistentVector #+cljs cljs.core/PersistentVector
+  (append-all [structure elements]
+    (reduce conj structure elements))
+  (prepend-all [structure elements]
+    (let [ret (transient [])]
+      (as-> ret <>
+            (reduce conj! <> elements)
+            (reduce conj! <> structure)
+            (persistent! <>)
+            )))
+
+  #+clj Object #+cljs default
+  (append-all [structure elements]
+    (concat structure elements))
+  (prepend-all [structure elements]
+    (concat elements structure))
+  )
+
+
 (defprotocol UpdateExtremes
  (update-first [s afn])
  (update-last [s afn]))
@ -473,6 +497,14 @@
 (defn vec-count [v]
  (count v))

+#+clj
+(defn transient-vec-count [^clojure.lang.ITransientVector v]
+  (.count v))
+
+#+cljs
+(defn transient-vec-count [v]
+  (count v))
+
 (extend-protocol UpdateExtremes
  #+clj clojure.lang.PersistentVector #+cljs cljs.core/PersistentVector
  (update-first [v afn]
@ -512,6 +544,9 @@
  #+clj clojure.lang.IPersistentVector #+cljs cljs.core/PersistentVector
  (fast-empty? [v]
    (= 0 (vec-count v)))
+  #+clj clojure.lang.ITransientVector #+cljs cljs.core/TransientVector
+  (fast-empty? [v]
+    (= 0 (transient-vec-count v)))
  #+clj Object #+cljs default
  (fast-empty? [s]
    (empty? s))
@ -702,6 +737,49 @@
  (transform* [this structure next-fn]
    (all-transform structure next-fn)))

+(defprotocol MapValsTransformProtocol
+  (map-vals-transform [structure next-fn]))
+
+(defn map-vals-non-transient-transform [structure empty-map next-fn]
+  (reduce-kv
+    (fn [m k v]
+      (assoc m k (next-fn v)))
+    empty-map
+    structure))
+
+(extend-protocol MapValsTransformProtocol
+  #+clj clojure.lang.PersistentArrayMap #+cljs cljs.core/PersistentArrayMap
+  (map-vals-transform [structure next-fn]
+    (map-vals-non-transient-transform structure {} next-fn)
+    )
+
+  #+clj clojure.lang.PersistentTreeMap #+cljs cljs.core/PersistentTreeMap
+  (map-vals-transform [structure next-fn]
+    (map-vals-non-transient-transform structure (sorted-map) next-fn)
+    )
+
+  #+clj clojure.lang.PersistentHashMap #+cljs cljs.core/PersistentHashMap
+  (map-vals-transform [structure next-fn]
+    (persistent!
+      (reduce-kv
+        (fn [m k v]
+          (assoc! m k (next-fn v)))
+        (transient
+          #+clj clojure.lang.PersistentHashMap/EMPTY #+cljs cljs.core.PersistentHashMap.EMPTY
+          )
+        structure
+        )))
+
+  #+clj Object #+cljs default
+  (map-vals-transform [structure next-fn]
+    (reduce-kv
+      (fn [m k v]
+        (assoc m k (next-fn v)))
+      (empty structure)
+      structure))
+  )
+
+
 (deftype ValCollect [])

 (extend-protocol p/Collector
@ -764,6 +842,16 @@
    (next-fn structure)
    ))

+(deftype TransientEndNavigator [])
+
+(extend-protocol p/Navigator
+  TransientEndNavigator
+  (select* [this structure next-fn]
+    (next-fn []))
+  (transform* [this structure next-fn]
+    (let [res (next-fn [])]
+      (reduce conj! structure res))))
+
 (defn extract-basic-filter-fn [path]
  (cond (fn? path)
        path
--- a/src/clj/com/rpl/specter/transient.cljx
+++ b/src/clj/com/rpl/specter/transient.cljx
@ -0,0 +1,91 @@
+(ns com.rpl.specter.transient
+  #+cljs
+  (:require-macros [com.rpl.specter.macros
+                    :refer
+                    [defnav
+                     defpathedfn]])
+  (:use #+clj
+        [com.rpl.specter.macros :only
+         [defnav
+          defpathedfn]])
+  (:require [com.rpl.specter.impl :as i]
+            [com.rpl.specter :refer [subselect selected?]]))
+
+(defnav
+  ^{:doc "Navigates to the specified key of a transient collection,
+          navigating to nil if it doesn't exist."}
+  keypath!
+  [key]
+  (select* [this structure next-fn]
+    (next-fn (get structure key)))
+  (transform* [this structure next-fn]
+    (assoc! structure key (next-fn (get structure key)))))
+
+(def END!
+  "Navigates to an empty (persistent) vector at the end of a transient vector."
+  (i/comp-paths* [(i/->TransientEndNavigator)]))
+
+(defn- t-get-first
+  [tv]
+  (nth tv 0))
+
+(defn- t-get-last
+  [tv]
+  (nth tv (dec (i/transient-vec-count tv))))
+
+(defn- t-update-first
+  [tv next-fn]
+  (assoc! tv 0 (next-fn (nth tv 0))))
+
+(defn- t-update-last
+  [tv next-fn]
+  (let [i (dec (i/transient-vec-count tv))]
+    (assoc! tv i (next-fn (nth tv i)))))
+
+(def FIRST!
+  "Navigates to the first element of a transient vector."
+  (i/->PosNavigator t-get-first t-update-first))
+
+(def LAST!
+  "Navigates to the last element of a transient vector."
+  (i/->PosNavigator t-get-last t-update-last))
+
+#+clj
+(defn- select-keys-from-transient-map
+  "Selects keys from transient map, because built-in select-keys uses
+  `find` which is unsupported."
+  [m m-keys]
+  (loop [result {}
+         m-keys m-keys]
+    (if-not (seq m-keys)
+      result
+      (let [k (first m-keys)
+            ;; support Clojure 1.6 where contains? is broken on transients
+            item (get m k ::not-found)]
+        (recur (if-not (identical? item ::not-found)
+                 (assoc result k item)
+                 result)
+               (rest m-keys))))))
+
+#+cljs
+(defn- select-keys-from-transient-map
+  "Uses select-keys on a transient map."
+  [m m-keys]
+  (select-keys m m-keys))
+
+(defnav
+  ^{:doc "Navigates to the specified persistent submap of a transient map."}
+  submap!
+  [m-keys]
+  (select* [this structure next-fn]
+    (next-fn (select-keys-from-transient-map structure m-keys)))
+  (transform* [this structure next-fn]
+    (let [selected (select-keys-from-transient-map structure m-keys)
+          res (next-fn selected)]
+      (as-> structure %
+        (reduce (fn [m k]
+                  (dissoc! m k))
+                % m-keys)
+        (reduce-kv (fn [m k v]
+                     (assoc! m k v))
+                   % res)))))
--- a/test/com/rpl/specter/core_test.cljx
+++ b/test/com/rpl/specter/core_test.cljx
@ -25,6 +25,7 @@
            #+cljs [cljs.test.check.generators :as gen]
            #+cljs [cljs.test.check.properties :as prop :include-macros true]
            [com.rpl.specter :as s]
+            [com.rpl.specter.transient :as t]
            [clojure.set :as set]))

 ;;TODO:
@ -64,8 +65,9 @@

 (defspec select-all-on-map
  (for-all+
-    [m (limit-size 5 (gen/map gen/keyword gen/int))]
-    (= (select [s/ALL s/LAST] m)
+    [m (limit-size 5 (gen/map gen/keyword gen/int))
+     p (gen/elements [s/MAP-VALS [s/ALL s/LAST]])]
+    (= (select p m)
       (for [[k v] m] v))
    ))

@ -81,8 +83,9 @@

 (defspec transform-all-on-map
  (for-all+
-    [m (limit-size 5 (gen/map gen/keyword gen/int))]
-    (= (transform [s/ALL s/LAST] inc m)
+    [m (limit-size 5 (gen/map gen/keyword gen/int))
+     p (gen/elements [s/MAP-VALS [s/ALL s/LAST]])]
+    (= (transform p inc m)
       (into {} (for [[k v] m] [k (inc v)]))
       )))

@ -1036,3 +1039,34 @@
    (is (= 2 (key e)))
    (is (= 4 (val e)))
    ))
+
+(defspec transient-vector-test
+  (for-all+
+    [v (gen/vector (limit-size 5 gen/int))]
+    (every? identity
+            (for [[path transient-path f]
+                  [[s/FIRST t/FIRST! (fnil inc 0)]  ;; fnil in case vector is empty
+                   [s/LAST t/LAST! (fnil inc 0)]
+                   [(s/keypath 0) (t/keypath! 0) (fnil inc 0)]
+                   [s/END t/END! #(conj % 1 2 3)]]]
+              (and (= (s/transform* path f v)
+                      (persistent! (s/transform* transient-path f (transient v))))
+                   (= (s/select* path v)
+                      (s/select* transient-path (transient v))))))))
+
+(defspec transient-map-test
+  (for-all+
+    [m (limit-size 5 (gen/not-empty (gen/map gen/keyword gen/int)))
+     new-key gen/keyword]
+    (let [existing-key (first (keys m))]
+      (every? identity
+              (for [[path transient-path f]
+                    [[(s/keypath existing-key) (t/keypath! existing-key) inc]
+                     [(s/keypath new-key) (t/keypath! new-key) (constantly 3)]
+                     [(s/submap [existing-key new-key])
+                      (t/submap! [existing-key new-key])
+                      (constantly {new-key 1234})]]]
+                (and (= (s/transform* path f m)
+                        (persistent! (s/transform* transient-path f (transient m))))
+                     (= (s/select* path m)
+                        (s/select* transient-path (transient m)))))))))