Merge pull request #120 from metosin/encode-decode

Encode decode

CHANGELOG.md

@@ -5,6 +5,8 @@
 * **BREAKING**: the router option key to extract body format has been renamed: `:extract-request-format` => `:reitit.coercion/extract-request-format`
   * should only concern you if you are not using [Muuntaja](https://github.com/metosin/muuntaja).
 * the `r/routes` returns just the path + data tuples as documented, not the compiled route results. To get the compiled results, use `r/compiled-routes` instead.
+* new [faster](https://github.com/metosin/reitit/blob/master/perf-test/clj/reitit/impl_perf_test.clj) and more correct encoders and decoders for query & path params.
+  * query-parameters are encoded with `reitit.impl/form-encode`, so spaces are `+` instead of `%20`.
 * welcome route name conflict resolution! If router has routes with same names, router can't be created. fix 'em.
 * sequential child routes are allowed, enabling this:
 

modules/reitit-core/src/reitit/impl.cljc

@@ -162,19 +162,46 @@
 (defn strip-nils [m]
   (->> m (remove (comp nil? second)) (into {})))
 
+#?(:clj (def +percents+ (into [] (map #(format "%%%02X" %) (range 0 256)))))
+
+#?(:clj (defn byte->percent [byte]
+          (nth +percents+ (if (< byte 0) (+ 256 byte) byte))))
+
+#?(:clj (defn percent-encode [^String s]
+          (->> (.getBytes s "UTF-8") (map byte->percent) (str/join))))
+
 ;;
-;; Path-parameters, see https://github.com/metosin/reitit/issues/75
+;; encoding & decoding
 ;;
 
+;; + is safe, but removed so it would work the same as with js
 (defn url-encode [s]
-  (some-> s
+  (if s
-          #?(:clj  (URLEncoder/encode "UTF-8")
+    #?(:clj  (str/replace s #"[^A-Za-z0-9\!'\(\)\*_~.-]+" percent-encode)
-             :cljs (js/encodeURIComponent))
+       :cljs (js/encodeURIComponent s))))
-          #?(:clj (.replace "+" "%20"))))
 
 (defn url-decode [s]
-  (some-> s #?(:clj  (URLDecoder/decode "UTF-8")
+  (if s
-               :cljs (js/decodeURIComponent))))
+    #?(:clj  (if (.contains ^String s "%")
+               (URLDecoder/decode
+                 (if (.contains ^String s "+")
+                   (.replace ^String s "+" "%2B")
+                   s)
+                 "UTF-8")
+               s)
+       :cljs (js/decodeURIComponent s))))
+
+(defn form-encode [s]
+  (if s
+    #?(:clj  (URLEncoder/encode ^String s "UTF-8")
+       :cljs (str/replace (js/encodeURIComponent s) "%20" "+"))))
+
+(defn form-decode [s]
+  (if s
+    #?(:clj  (if (or (.contains ^String s "%") (.contains ^String s "+"))
+               (URLDecoder/decode ^String s "UTF-8")
+               s)
+       :cljs (js/decodeURIComponent (str/replace s "+" " ")))))
 
 (defprotocol IntoString
   (into-string [_]))
@@ -203,7 +230,7 @@
   (into-string [this] (str this))
 
   nil
-  (into-string [this]))
+  (into-string [_]))
 
 (defn path-params
   "shallow transform of the path parameters values into strings"
@@ -219,9 +246,9 @@
   [params]
   (->> params
        (map (fn [[k v]]
-              (str (url-encode (into-string k))
+              (str (form-encode (into-string k))
                    "="
-                   (url-encode (into-string v)))))
+                   (form-encode (into-string v)))))
        (str/join "&")))
 
 (defmacro goog-extend [type base-type ctor & methods]
@@ -231,7 +258,7 @@
      (goog/inherits ~type ~base-type)
 
      ~@(map
-        (fn [method]
+         (fn [method]
-          `(set! (.. ~type -prototype ~(symbol (str "-" (first method))))
+           `(set! (.. ~type -prototype ~(symbol (str "-" (first method))))
-                 (fn ~@(rest method))))
+                  (fn ~@(rest method))))
-        methods)))
+         methods)))

perf-test/clj/reitit/impl_perf_test.clj

@@ -0,0 +1,170 @@
+(ns reitit.impl-perf-test
+  (:require [criterium.core :as cc]
+            [reitit.perf-utils :refer :all]
+            [ring.util.codec]
+            [reitit.impl])
+  (:import (java.net URLDecoder URLEncoder)))
+
+;;
+;; start repl with `lein perf repl`
+;; perf measured with the following setup:
+;;
+;; Model Name:            MacBook Pro
+;; Model Identifier:      MacBookPro11,3
+;; Processor Name:        Intel Core i7
+;; Processor Speed:       2,5 GHz
+;; Number of Processors:  1
+;; Total Number of Cores: 4
+;; L2 Cache (per Core):   256 KB
+;; L3 Cache:              6 MB
+;; Memory:                16 GB
+;;
+
+
+(defn test! [f input]
+  (do
+    (println "\u001B[33m")
+    (println (pr-str input) "=>" (pr-str (f input)))
+    (println "\u001B[0m")
+    (cc/quick-bench (f input))))
+
+(defn url-decode-naive [s]
+  (URLDecoder/decode
+    (.replace ^String s "+" "%2B")
+    "UTF-8"))
+
+(defn url-decode! []
+
+  ;; ring
+
+  ;; 890ns
+  ;; 190ns
+  ;; 90ns
+  ;; 80ns
+
+  ;; naive
+
+  ;; 750ns
+  ;; 340ns
+  ;; 420ns
+  ;; 200ns
+
+  ;; reitit
+
+  ;; 630ns (-29%)
+  ;; 12ns (-94%)
+  ;; 8ns (-91%)
+  ;; 8ns (-90%)
+
+  (doseq [fs ['ring.util.codec/url-decode
+              'url-decode-naive
+              'reitit.impl/url-decode]
+          :let [f (deref (resolve fs))]]
+    (suite (str fs))
+    (doseq [s ["aja%20hiljaa+sillalla"
+               "aja_hiljaa_sillalla"
+               "1+1"
+               "1"]]
+      (test! f s))))
+
+(defn url-encode-naive [^String s]
+  (cond-> (.replace (URLEncoder/encode s "UTF-8") "+" "%20")
+          (.contains s "+") (.replace "%2B" "+")
+          (.contains s "~") (.replace "%7E" "~")
+          (.contains s "=") (.replace "%3D" "=")
+          (.contains s "!") (.replace "%21" "!")
+          (.contains s "'") (.replace "%27" "'")
+          (.contains s "(") (.replace "%28" "(")
+          (.contains s ")") (.replace "%29" ")")))
+
+(defn url-encode! []
+
+  ;; ring
+
+  ;; 2500ns
+  ;; 610ns
+  ;; 160ns
+  ;; 120ns
+
+  ;; naive
+
+  ;; 1000ns
+  ;; 440ns
+  ;; 570ns
+  ;; 200ns
+
+  ;; reitit
+
+  ;; 1400ns
+  ;; 740ns
+  ;; 180ns
+  ;; 130ns
+
+  (doseq [fs ['ring.util.codec/url-encode
+              'url-encode-naive
+              'reitit.impl/url-encode]
+          :let [f (deref (resolve fs))]]
+    (suite (str fs))
+    (doseq [s ["aja hiljaa+sillalla"
+               "aja_hiljaa_sillalla"
+               "1+1"
+               "1"]]
+      (test! f s))))
+
+(defn form-decode! []
+
+  ;; ring
+
+  ;; 280ns
+  ;; 130ns
+  ;; 43ns
+  ;; 25ns
+
+  ;; reitit
+
+  ;; 270ns (-4%)
+  ;; 20ns (-84%)
+  ;; 47ns (+8%)
+  ;; 12ns (-52%)
+
+  (doseq [fs ['ring.util.codec/form-decode-str
+              'reitit.impl/form-decode]
+          :let [f (deref (resolve fs))]]
+    (suite (str fs))
+    (doseq [s ["%2Baja%20hiljaa+sillalla"
+               "aja_hiljaa_sillalla"
+               "1+1"
+               "1"]]
+      (test! f s))))
+
+(defn form-encode! []
+
+  ;; ring
+
+  ;; 240ns
+  ;; 120ns
+  ;; 130ns
+  ;; 31ns
+
+  ;; reitit
+
+  ;; 210ns
+  ;; 120ns
+  ;; 130ns
+  ;; 30ns
+
+  (doseq [fs ['ring.util.codec/form-encode
+              'reitit.impl/form-encode]
+          :let [f (deref (resolve fs))]]
+    (suite (str fs))
+    (doseq [s ["aja hiljaa+sillalla"
+               "aja_hiljaa_sillalla"
+               "1+1"
+               "1"]]
+      (test! f s))))
+
+(comment
+  (url-decode!)
+  (url-encode!)
+  (form-decode!)
+  (form-encode!))

test/cljc/reitit/impl_test.cljc

@@ -57,10 +57,117 @@
     {:a 1} "a=1"
     {:a nil} "a="
     {:a :b :c "d"} "a=b&c=d"
-    {:a "b c"} "a=b%20c"))
+    {:a "b c"} "a=b+c"))
 
 ; TODO: support seq values?
 ;{:a ["b" "c"]} "a=b&a=c"
 ;{:a ["c" "b"]} "a=c&a=b"
 ;{:a (seq [1 2])} "a=1&a=2"
 ;{:a #{"c" "b"}} "a=b&a=c"
+
+;; test from https://github.com/playframework/playframework -> UriEncodingSpec.scala
+
+(deftest url-encode-test
+  (are [in out]
+    (= out (impl/url-encode in))
+
+    "/" "%2F"
+    "?" "%3F"
+    "#" "%23"
+    "[" "%5B"
+    "]" "%5D"
+    "!" "!"
+    #_#_"$" "$"
+    #_#_"&" "&"
+    "'" "'"
+    "(" "("
+    ")" ")"
+    "*" "*"
+    #_#_"+" "+"
+    #_#_"," ","
+    #_#_";" ";"
+    #_#_"=" "="
+    #_#_":" ":"
+    #_#_"@" "@"
+    "a" "a"
+    "z" "z"
+    "A" "A"
+    "Z" "Z"
+    "0" "0"
+    "9" "9"
+    "-" "-"
+    "." "."
+    "_" "_"
+    "~" "~"
+    "\000" "%00"
+    "\037" "%1F"
+    " " "%20"
+    "\"" "%22"
+    "%" "%25"
+    "<" "%3C"
+    ">" "%3E"
+    "\\" "%5C"
+    "^" "%5E"
+    "`" "%60"
+    "{" "%7B"
+    "|" "%7C"
+    "}" "%7D"
+    "\177" "%7F"
+    #_#_"\377" "%FF"
+
+    "£0.25" "%C2%A30.25"
+    "€100" "%E2%82%AC100"
+    "«küßî»" "%C2%ABk%C3%BC%C3%9F%C3%AE%C2%BB"
+    "“ЌύБЇ”" "%E2%80%9C%D0%8C%CF%8D%D0%91%D0%87%E2%80%9D"
+
+    "\000" "%00"
+    #_#_"\231" "%99"
+    #_#_"\252" "%AA"
+    #_#_"\377" "%FF"
+
+    "" ""
+    "1" "1"
+    "12" "12"
+    "123" "123"
+    "1234567890" "1234567890"
+
+    "Hello world" "Hello%20world"
+    "/home/foo" "%2Fhome%2Ffoo"
+
+    " " "%20"
+    "+" "%2B" #_"+"
+    " +" "%20%2B" #_"%20+"
+    #_#_"1+2=3" "1+2=3"
+    #_#_"1 + 2 = 3" "1%20+%202%20=%203"))
+
+(deftest url-decode-test
+  (are [in out]
+    (= out (impl/url-decode in))
+
+    "1+1" "1+1"
+    "%21" "!"
+    "%61" "a"
+    "%31%32%33" "123"
+    "%2b" "+"
+    "%7e" "~"
+    "hello%20world" "hello world"
+    "a%2fb" "a/b"
+    "a/.." "a/.."
+    "a/." "a/."
+    "//a" "//a"
+    "a//b" "a//b"
+    "a//" "a//"
+    "/path/%C2%ABk%C3%BC%C3%9F%C3%AE%C2%BB" "/path/«küßî»"
+    "/path/%E2%80%9C%D0%8C%CF%8D%D0%91%D0%87%E2%80%9D" "/path/“ЌύБЇ”"))
+
+(deftest form-encode-test
+  (are [in out]
+    (= out (impl/form-encode in))
+
+    "+632 905 123 4567" "%2B632+905+123+4567"))
+
+(deftest form-decode-test
+  (are [in out]
+    (= out (impl/form-decode in))
+
+    "%2B632+905+123+4567" "+632 905 123 4567"))