From f2d131a897230d5485949ab8b5f28f985db644fe Mon Sep 17 00:00:00 2001 From: Tommi Reiman Date: Sat, 26 Jan 2019 16:03:44 +0200 Subject: [PATCH] wip --- examples/ring-swagger/src/example/server.clj | 3 +- modules/reitit-core/java-src/reitit/Trie.java | 216 ++++++++++++++++++ modules/reitit-core/project.clj | 2 +- modules/reitit-core/src/reitit/core.cljc | 7 +- modules/reitit-core/src/reitit/trie.cljc | 146 ++++++++++++ perf-test/clj/reitit/bide_perf_test.clj | 66 ++++++ .../clj/reitit/opensensors_perf_test.clj | 21 +- .../clj/reitit/prefix_tree_perf_test.clj | 25 +- project.clj | 3 +- 9 files changed, 469 insertions(+), 20 deletions(-) create mode 100644 modules/reitit-core/java-src/reitit/Trie.java create mode 100644 modules/reitit-core/src/reitit/trie.cljc diff --git a/examples/ring-swagger/src/example/server.clj b/examples/ring-swagger/src/example/server.clj index fb8e869d..c79f0db2 100644 --- a/examples/ring-swagger/src/example/server.clj +++ b/examples/ring-swagger/src/example/server.clj @@ -83,7 +83,8 @@ (ring/routes (swagger-ui/create-swagger-ui-handler {:path "/" - :config {:validatorUrl nil}}) + :config {:validatorUrl nil + :operationsSorter "alpha"}}) (ring/create-default-handler)))) (defn start [] diff --git a/modules/reitit-core/java-src/reitit/Trie.java b/modules/reitit-core/java-src/reitit/Trie.java new file mode 100644 index 00000000..4dfbae2b --- /dev/null +++ b/modules/reitit-core/java-src/reitit/Trie.java @@ -0,0 +1,216 @@ +package reitit; + +// https://www.codeproject.com/Tips/1190293/Iteration-Over-Java-Collections-with-High-Performa + +import clojure.lang.Keyword; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.*; + +public class Trie { + + private static String decode(char[] chars, int i, int j, boolean hasPercent, boolean hasPlus) { + final String s = new String(chars, i, j); + try { + if (hasPercent) { + return URLDecoder.decode(hasPlus ? s.replace("+", "%2B") : s, "UTF-8"); + } + } catch (UnsupportedEncodingException ignored) { + } + return s; + } + + public static class Match { + public final List params = new ArrayList<>(); + public Object data; + + @Override + public String toString() { + Map m = new HashMap<>(); + m.put(Keyword.intern("data"), data); + m.put(Keyword.intern("params"), params); + return m.toString(); + } + } + + public static class Path { + final char[] value; + final int size; + + Path(String value) { + this.value = value.toCharArray(); + this.size = value.length(); + } + } + + public interface Matcher { + Match match(int i, Path path, Match match); + } + + public static StaticMatcher staticMatcher(String path, Matcher child) { + return new StaticMatcher(path, child); + } + + static class StaticMatcher implements Matcher { + private final Matcher child; + private final char[] path; + private final int size; + + StaticMatcher(String path, Matcher child) { + this.path = path.toCharArray(); + this.size = path.length(); + this.child = child; + } + + @Override + public Match match(int i, Path path, Match match) { + final char[] value = path.value; + if (path.size < i + size) { + return null; + } + for (int j = 0; j < size; j++) { + if (value[j + i] != this.path[j]) { + return null; + } + } + return child.match(i + size, path, match); + } + + @Override + public String toString() { + return "[\"" + new String(path) + "\" " + child + "]"; + } + } + + public static DataMatcher dataMatcher(Object data) { + return new DataMatcher(data); + } + + static final class DataMatcher implements Matcher { + private final Object data; + + DataMatcher(Object data) { + this.data = data; + } + + @Override + public Match match(int i, Path path, Match match) { + if (i == path.size) { + match.data = data; + return match; + } + return null; + } + + @Override + public String toString() { + return (data != null ? data.toString() : "nil"); + } + } + + public static WildMatcher wildMatcher(Keyword parameter, Matcher child) { + return new WildMatcher(parameter, child); + } + + static final class WildMatcher implements Matcher { + private final Keyword key; + private final Matcher child; + + WildMatcher(Keyword key, Matcher child) { + this.key = key; + this.child = child; + } + + @Override + public Match match(int i, Path path, Match match) { + final char[] value = path.value; + if (i < path.size && value[i] != '/') { + boolean hasPercent = false; + boolean hasPlus = false; + for (int j = i; j < path.size; j++) { + if (value[j] == '/') { + final Match m = child.match(j, path, match); + if (m != null) { + m.params.add(key); + m.params.add(decode(value, i, j - i, hasPercent, hasPlus)); + } + return m; + } else if (value[j] == '%') { + hasPercent = true; + } else if (value[j] == '+') { + hasPlus = true; + } + } + if (child instanceof DataMatcher) { + final Match m = child.match(path.size, path, match); + m.params.add(key); + m.params.add(decode(value, i, path.size - i, hasPercent, hasPlus)); + return m; + } + } + return null; + } + + @Override + public String toString() { + return "[" + key + " " + child + "]"; + } + } + + public static LinearMatcher linearMatcher(List childs) { + return new LinearMatcher(childs); + } + + static final class LinearMatcher implements Matcher { + + private final Matcher[] childs; + private final int size; + + LinearMatcher(List childs) { + this.childs = childs.toArray(new Matcher[0]); + this.size = childs.size(); + } + + @Override + public Match match(int i, Path path, Match match) { + for (int j = 0; j < size; j++) { + final Match m = childs[j].match(i, path, match); + if (m != null) { + return m; + } + } + return null; + } + + @Override + public String toString() { + return Arrays.toString(childs); + } + } + + public static Object lookup(Matcher matcher, String path) { + return matcher.match(0, new Path(path), new Match()); + } + + public static void main(String[] args) { + + //Matcher matcher = new StaticMatcher("/kikka", new StaticMatcher("/kukka", new DataMatcher(1))); +// Matcher matcher = +// staticMatcher("/kikka/", +// wildMatcher(Keyword.intern("kukka"), +// staticMatcher("/kikka", +// dataMatcher(1)))); + Matcher matcher = + linearMatcher( + Arrays.asList( + staticMatcher("/auth/", + linearMatcher( + Arrays.asList( + staticMatcher("login", dataMatcher(1)), + staticMatcher("recovery", dataMatcher(2))))))); + System.err.println(matcher); + System.out.println(lookup(matcher, "/auth/login")); + System.out.println(lookup(matcher, "/auth/recovery")); + } +} diff --git a/modules/reitit-core/project.clj b/modules/reitit-core/project.clj index afcb84b8..4f04c95a 100644 --- a/modules/reitit-core/project.clj +++ b/modules/reitit-core/project.clj @@ -8,5 +8,5 @@ :plugins [[lein-parent "0.3.2"]] :parent-project {:path "../../project.clj" :inherit [:deploy-repositories :managed-dependencies]} - :java-source-paths ["java-src"] + ;:java-source-paths ["java-src"] :dependencies [[meta-merge]]) diff --git a/modules/reitit-core/src/reitit/core.cljc b/modules/reitit-core/src/reitit/core.cljc index b905915b..5df916af 100644 --- a/modules/reitit-core/src/reitit/core.cljc +++ b/modules/reitit-core/src/reitit/core.cljc @@ -2,6 +2,7 @@ (:require [meta-merge.core :refer [meta-merge]] [clojure.string :as str] [reitit.segment :as segment] + [reitit.segment :as trie] [reitit.impl :as impl #?@(:cljs [:refer [Route]])]) #?(:clj (:import (reitit.impl Route)))) @@ -265,11 +266,11 @@ f #(if-let [path (impl/path-for route %)] (->Match p data result (impl/url-decode-coll %) path) (->PartialMatch p data result % path-params))] - [(segment/insert pl p (->Match p data result nil nil)) + [(trie/insert pl p (->Match p data result nil nil)) (if name (assoc nl name f) nl)])) [nil {}] compiled-routes) - pl (segment/compile pl) + pl (trie/compile pl) lookup (impl/fast-map nl) routes (uncompile-routes compiled-routes)] ^{:type ::router} @@ -286,7 +287,7 @@ (route-names [_] names) (match-by-path [_ path] - (if-let [match (segment/lookup pl path)] + (if-let [match (trie/lookup pl path)] (-> (:data match) (assoc :path-params (:path-params match)) (assoc :path path)))) diff --git a/modules/reitit-core/src/reitit/trie.cljc b/modules/reitit-core/src/reitit/trie.cljc new file mode 100644 index 00000000..1fa98100 --- /dev/null +++ b/modules/reitit-core/src/reitit/trie.cljc @@ -0,0 +1,146 @@ +(ns reitit.trie + (:refer-clojure :exclude [compile]) + (:require [clojure.string :as str]) + (:import [reitit Trie Trie$Match Trie$Matcher])) + +(defrecord Match [data path-params]) +(defrecord Node [children wilds catch-all data]) + +;; https://stackoverflow.com/questions/8033655/find-longest-common-prefix +(defn- -common-prefix [s1 s2] + (let [max (min (count s1) (count s2))] + (loop [i 0] + (cond + ;; full match + (> i max) + (subs s1 0 max) + ;; partial match + (not= (get s1 i) (get s2 i)) + (if-not (zero? i) (subs s1 0 i)) + ;; recur + :else + (recur (inc i)))))) + +(defn- -keyword [s] + (if-let [i (str/index-of s "/")] + (keyword (subs s 0 i) (subs s (inc i))) + (keyword s))) + +(defn- -split [s] + (let [-static (fn [from to] (if-not (= from to) [(subs s from to)])) + -wild (fn [from to] [(-keyword (subs s (inc from) to))]) + -catch-all (fn [from to] [#{(keyword (subs s (inc from) to))}])] + (loop [ss nil, from 0, to 0] + (if (= to (count s)) + (concat ss (-static from to)) + (condp = (get s to) + \{ (let [to' (or (str/index-of s "}" to) (throw (ex-info (str "Unbalanced brackets: " (pr-str s)) {})))] + (recur (concat ss (-static from to) (-wild to to')) (inc to') (inc to'))) + \: (let [to' (or (str/index-of s "/" to) (count s))] + (recur (concat ss (-static from to) (-wild to to')) to' to')) + \* (let [to' (count s)] + (recur (concat ss (-static from to) (-catch-all to to')) to' to')) + (recur ss from (inc to))))))) + +(defn- -node [m] + (map->Node (merge {:children {}, :wilds {}, :catch-all {}} m))) + +(defn- -insert [node [path & ps] data] + (let [node' (cond + + (nil? path) + (assoc node :data data) + + (keyword? path) + (update-in node [:wilds path] (fn [n] (-insert (or n (-node {})) ps data))) + + (set? path) + (assoc-in node [:catch-all path] (-node {:data data})) + + (str/blank? path) + (-insert node ps data) + + :else + (or + (reduce + (fn [_ [p n]] + (if-let [cp (-common-prefix p path)] + (if (= cp p) + ;; insert into child node + (let [n' (-insert n (conj ps (subs path (count p))) data)] + (reduced (assoc-in node [:children p] n'))) + ;; split child node + (let [rp (subs p (count cp)) + rp' (subs path (count cp)) + n' (-insert (-node {}) ps data) + n'' (-insert (-node {:children {rp n, rp' n'}}) nil nil)] + (reduced (update node :children (fn [children] + (-> children + (dissoc p) + (assoc cp n''))))))))) + nil (:children node)) + ;; new child node + (assoc-in node [:children path] (-insert (-node {}) ps data))))] + (if-let [child (get-in node' [:children ""])] + ;; optimize by removing empty paths + (-> (merge-with merge node' child) + (update :children dissoc "")) + node'))) + +(defn insert [node path data] + (-insert (or node (-node {})) (-split path) data)) + +(defn ^Trie$Matcher compile [{:keys [data children wilds catch-all]}] + (let [matchers (cond-> [] + data (conj (Trie/dataMatcher data)) + children (into (for [[p c] children] (Trie/staticMatcher p (compile c)))) + wilds (into (for [[p c] wilds] (Trie/wildMatcher p (compile c)))))] + (if (rest matchers) + (Trie/linearMatcher matchers) + (first matchers)))) + +(defn pretty [{:keys [data children wilds catch-all]}] + (into + (if data [data] []) + (mapcat (fn [[p n]] [p (pretty n)]) (concat children wilds catch-all)))) + +(defn lookup [^Trie$Matcher matcher path] + (if-let [match ^Trie$Match (Trie/lookup matcher ^String path)] + (->Match (.data match) (clojure.lang.PersistentHashMap/create (.toArray (.params match)))))) + +;; +;; matcher +;; + +;; +;; spike +;; + +(-> nil + (insert "/:abba" 1) + (insert "/kikka" 2) + (insert "/kikka/kakka/kukka" 3) + (insert "/kikka/:kakka/kukka" 4) + (insert "/kikka/kuri/{user/doc}.html" 5) + (insert "/kikkare/*path" 6) + #_(pretty)) + +(-> nil + (insert "/kikka" 2) + (insert "/kikka/kakka/kukka" 3) + (insert "/kikka/:kakka/kurkku" 4) + (insert "/kikka/kuri/{user/doc}/html" 5) + (compile) + (lookup "/kikka/kakka/kurkku")) + +;; => + +["/" + ["kikka" [2 + "/" ["k" ["akka/kukka" [3] + "uri/" [:user/doc [".html" [5]]]] + :kakka ["/kukka" [4]]] + "re/" [#{:path} [6]]] + :abba [1]]] + + diff --git a/perf-test/clj/reitit/bide_perf_test.clj b/perf-test/clj/reitit/bide_perf_test.clj index b96d971d..885a32bc 100644 --- a/perf-test/clj/reitit/bide_perf_test.clj +++ b/perf-test/clj/reitit/bide_perf_test.clj @@ -205,3 +205,69 @@ (routing-test1) (routing-test2) (reverse-routing-test)) + +(import '[reitit Trie]) + +(set! *warn-on-reflection* true) + +(comment + (let [trie ] + + + (println + (Trie/lookup trie "/auth/login")) + + ;; 27ns + (cc/quick-bench + (dotimes [_ 1000] + (Trie/lookup trie "/auth/login"))) + + (println + (Trie/lookup trie "/auth/recovery/token/123")) + + ;; 82ns + (cc/quick-bench + (dotimes [_ 1000] + (Trie/lookup trie "/auth/recovery/token/123"))) + + (println + (Trie/lookup trie "/workspace/1/1")) + + ;; 96ns + (cc/quick-bench + (dotimes [_ 1000] + (Trie/lookup trie "/workspace/1/1"))))) + +(comment + (let [trie (Trie/linearMatcher + [(Trie/staticMatcher + "/auth/" (Trie/linearMatcher + [(Trie/staticMatcher "login" (Trie/dataMatcher 1)) + (Trie/staticMatcher "recovery/token/" (Trie/wildMatcher :token (Trie/dataMatcher 2)))])) + (Trie/staticMatcher + "/workspace/" (Trie/wildMatcher :project (Trie/staticMatcher "/" (Trie/wildMatcher :page (Trie/dataMatcher 3)))))])] + + + (println + (Trie/lookup trie "/auth/login")) + + ;; 27ns + (cc/quick-bench + (dotimes [_ 1000] + (Trie/lookup trie "/auth/login"))) + + (println + (Trie/lookup trie "/auth/recovery/token/123")) + + ;; 82ns + (cc/quick-bench + (dotimes [_ 1000] + (Trie/lookup trie "/auth/recovery/token/123"))) + + (println + (Trie/lookup trie "/workspace/1/1")) + + ;; 96ns + (cc/quick-bench + (dotimes [_ 1000] + (Trie/lookup trie "/workspace/1/1"))))) diff --git a/perf-test/clj/reitit/opensensors_perf_test.clj b/perf-test/clj/reitit/opensensors_perf_test.clj index 427d9b2e..f8ce7900 100644 --- a/perf-test/clj/reitit/opensensors_perf_test.clj +++ b/perf-test/clj/reitit/opensensors_perf_test.clj @@ -578,35 +578,36 @@ ;; 806ns (decode path-parameters) ;; 735ns (maybe-map-values) ;; 474ns (java-segment-router) - (b! "reitit-ring" reitit-ring-f) + #_(b! "reitit-ring" reitit-ring-f) ;; 385ns (java-segment-router, no injects) - (b! "reitit-ring-fast" reitit-ring-fast-f) + #_(b! "reitit-ring-fast" reitit-ring-fast-f) ;; 2553ns (linear-router) ;; 630ns (segment-router-backed) - (b! "reitit-ring-linear" reitit-ring-linear-f) + #_(b! "reitit-ring-linear" reitit-ring-linear-f) ;; 2137ns - (b! "calfpath-walker" calfpath-walker-f) + #_(b! "calfpath-walker" calfpath-walker-f) ;; 4774ns - (b! "calfpath-unroll" calfpath-unroll-f) + #_(b! "calfpath-unroll" calfpath-unroll-f) ;; 2821ns - (b! "pedestal" pedestal-f) + #_(b! "pedestal" pedestal-f) ;; 4803ns - (b! "calfpath-macros" calfpath-macros-f) + #_(b! "calfpath-macros" calfpath-macros-f) ;; 11615ns - (b! "compojure" compojure-f) + #_(b! "compojure" compojure-f) ;; 15034ns - (b! "bidi" bidi-f) + #_(b! "bidi" bidi-f) ;; 19688ns - (b! "ataraxy" ataraxy-f))) + #_(b! "ataraxy" ataraxy-f))) (comment (bench-rest!)) + diff --git a/perf-test/clj/reitit/prefix_tree_perf_test.clj b/perf-test/clj/reitit/prefix_tree_perf_test.clj index aa266152..b56f2c58 100644 --- a/perf-test/clj/reitit/prefix_tree_perf_test.clj +++ b/perf-test/clj/reitit/prefix_tree_perf_test.clj @@ -2,6 +2,7 @@ (:require [clojure.test :refer :all] [io.pedestal.http.route.prefix-tree :as p] [reitit.segment :as segment] + [reitit.trie :as trie] [criterium.core :as cc]) (:import (reitit SegmentTrie))) @@ -70,7 +71,7 @@ (p/insert acc p d)) nil routes)) -(def matcher +(def segment-matcher (.matcher ^SegmentTrie (reduce @@ -78,6 +79,13 @@ (segment/insert acc p d)) nil routes))) +(def trie-matcher + (trie/compile + (reduce + (fn [acc [p d]] + (trie/insert acc p d)) + nil routes))) + (defn bench! [] ;; 2.3µs @@ -110,12 +118,21 @@ ;; 0.51µs (Cleanup) ;; 0.30µs (Java) (cc/quick-bench - (segment/lookup matcher "/v1/orgs/1/topics"))) + (segment/lookup segment-matcher "/v1/orgs/1/topics")) + + ;; 0.32µs (initial) + ;; 0.30µs (iterate arrays) + ;; 0.28µs (list-params) + (cc/quick-bench + (trie/lookup trie-matcher "/v1/orgs/1/topics"))) (comment (bench!)) +(set! *warn-on-reflection* true) + (comment (p/lookup pedestal-tree "/v1/orgs/1/topics") - #_(trie/lookup reitit-tree "/v1/orgs/1/topics" {}) - (segment/lookup matcher "/v1/orgs/1/topics")) + (trie/lookup trie-matcher "/v1/orgs/1/topics") + (segment/lookup segment-matcher "/v1/orgs/1/topics")) + diff --git a/project.clj b/project.clj index 19ea2c08..f67ae638 100644 --- a/project.clj +++ b/project.clj @@ -38,6 +38,7 @@ [io.pedestal/pedestal.service "0.5.5"]] :plugins [[jonase/eastwood "0.3.4"] + [lein-virgil "0.1.7"] [lein-doo "0.1.11"] [lein-cljsbuild "1.1.7"] [lein-cloverage "1.0.13"] @@ -61,7 +62,7 @@ "modules/reitit-sieppari/src" "modules/reitit-pedestal/src"] - :java-source-paths ["modules/reitit-core/java-src"] + ;:java-source-paths ["modules/reitit-core/java-src"] :dependencies [[org.clojure/clojure "1.10.0"] [org.clojure/clojurescript "1.10.439"]