luciano/monger
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #120 from Deraen/improve-from-db-object-perf

Browse source 
Improve from db object perf
This commit is contained in:
Michael Klishin 2015-10-09 00:58:42 +03:00
commit 2856631638
6 changed files with 40 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/clojure/monger/conversion.clj
View file

@ -105,7 +105,6 @@
(declare associate-pairs)
(defprotocol ConvertFromDBObject (defprotocol ConvertFromDBObject
(from-db-object [input keywordize] "Converts given DBObject instance to a piece of Clojure data")) (from-db-object [input keywordize] "Converts given DBObject instance to a piece of Clojure data"))
@ -116,10 +115,6 @@
Object Object
(from-db-object [input keywordize] input) (from-db-object [input keywordize] input)
Map
(from-db-object [^Map input keywordize]
(associate-pairs (.entrySet input) keywordize))
List List
(from-db-object [^List input keywordize] (from-db-object [^List input keywordize]
(vec (map #(from-db-object % keywordize) input))) (vec (map #(from-db-object % keywordize) input)))
@ -136,22 +131,13 @@
(from-db-object [^DBObject input keywordize] (from-db-object [^DBObject input keywordize]
;; DBObject provides .toMap, but the implementation in ;; DBObject provides .toMap, but the implementation in
;; subclass GridFSFile unhelpfully throws ;; subclass GridFSFile unhelpfully throws
;; UnsupportedOperationException. This part is taken from congomongo and ;; UnsupportedOperationException.
;; may need revisiting at a later point. MK.
(associate-pairs (for [key-set (.keySet input)] [key-set (.get input key-set)])
keywordize)))
(defn- associate-pairs [pairs keywordize]
;; Taking the keywordize test out of the fn reduces derefs
;; dramatically, which was the main barrier to matching pure-Java
;; performance for this marshalling. Taken from congomongo. MK.
(reduce (if keywordize (reduce (if keywordize
(fn [m [^String k v]] (fn [m ^String k]
(assoc m (keyword k) (from-db-object v true))) (assoc m (keyword k) (from-db-object (.get input k) true)))
(fn [m [^String k v]] (fn [m ^String k]
(assoc m k (from-db-object v false)))) (assoc m k (from-db-object (.get input k) false))))
{} (reverse pairs))) {} (.keySet input))))

6
test/monger/test/authentication_test.clj
View file

@ -12,12 +12,12 @@
(when-not (System/getenv "CI") (when-not (System/getenv "CI")
(deftest ^{:authentication true} connect-to-mongo-via-uri-without-credentials (deftest ^{:authentication true} connect-to-mongo-via-uri-without-credentials
(let [{:keys [conn db]} (mg/connect-via-uri "mongodb://127.0.0.1/monger-test4")] (let [{:keys [conn db]} (mg/connect-via-uri "mongodb://127.0.0.1/monger-test4")]
(is (= (-> conn .getAddress ^InetAddress (.sameHost "127.0.0.1")))))) (is (-> conn .getAddress (.sameHost "127.0.0.1")))))
(deftest ^{:authentication true} connect-to-mongo-via-uri-with-valid-credentials (deftest ^{:authentication true} connect-to-mongo-via-uri-with-valid-credentials
(let [{:keys [conn db]} (mg/connect-via-uri "mongodb://clojurewerkz/monger:monger@127.0.0.1/monger-test4")] (let [{:keys [conn db]} (mg/connect-via-uri "mongodb://clojurewerkz/monger:monger@127.0.0.1/monger-test4")]
(is (= "monger-test4" (.getName db))) (is (= "monger-test4" (.getName db)))
(is (= (-> conn .getAddress ^InetAddress (.sameHost "127.0.0.1")))) (is (-> conn .getAddress (.sameHost "127.0.0.1")))
(mc/remove db "documents") (mc/remove db "documents")
;; make sure that the database is selected ;; make sure that the database is selected
;; and operations get through. ;; and operations get through.
@ -27,7 +27,7 @@
(if-let [uri (System/getenv "MONGOHQ_URL")] (if-let [uri (System/getenv "MONGOHQ_URL")]
(deftest ^{:external true :authentication true} connect-to-mongo-via-uri-with-valid-credentials (deftest ^{:external true :authentication true} connect-to-mongo-via-uri-with-valid-credentials
(let [{:keys [conn db]} (mg/connect-via-uri uri)] (let [{:keys [conn db]} (mg/connect-via-uri uri)]
(is (= (-> conn .getAddress ^InetAddress (.sameHost "127.0.0.1"))))))) (is (-> conn .getAddress (.sameHost "127.0.0.1"))))))
;; ;;

6
test/monger/test/conversion_test.clj
View file

@ -112,20 +112,20 @@
(.put "name" name) (.put "name" name)
(.put "age" age)) (.put "age" age))
output (from-db-object input false)] output (from-db-object input false)]
(is (= (output { "name" name, "age" age }))) (is (= output { "name" name, "age" age }))
(is (= (output "name") name)) (is (= (output "name") name))
(is (nil? (output :name))) (is (nil? (output :name)))
(is (= (output "age") age)) (is (= (output "age") age))
(is (nil? (output "points"))))) (is (nil? (output "points")))))
(deftest convert-flat-db-object-to-map-without-keywordizing (deftest convert-flat-db-object-to-map-with-keywordizing
(let [name "Michael" (let [name "Michael"
age 26 age 26
input (doto (BasicDBObject.) input (doto (BasicDBObject.)
(.put "name" name) (.put "name" name)
(.put "age" age)) (.put "age" age))
output (from-db-object input true)] output (from-db-object input true)]
(is (= (output { :name name, :age age }))) (is (= output { :name name, :age age }))
(is (= (output :name) name)) (is (= (output :name) name))
(is (nil? (output "name"))) (is (nil? (output "name")))
(is (= (output :age) age)) (is (= (output :age) age))

15
test/monger/test/regular_finders_test.clj
View file

@ -8,7 +8,8 @@
[monger.result :as mgres] [monger.result :as mgres]
[monger.conversion :as mgcnv] [monger.conversion :as mgcnv]
[clojure.test :refer :all] [clojure.test :refer :all]
[monger.operators :refer :all])) [monger.operators :refer :all]
[monger.conversion :refer [to-db-object]]))
(let [conn (mg/connect) (let [conn (mg/connect)
db (mg/get-db conn "monger-test")] db (mg/get-db conn "monger-test")]
@ -147,36 +148,36 @@
doc-id (mu/random-uuid) doc-id (mu/random-uuid)
doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }] doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }]
(mc/insert db collection doc) (mc/insert db collection doc)
(is (= (doc (mc/find-by-id db collection doc-id)))))) (is (= (to-db-object doc) (mc/find-by-id db collection doc-id)))))
(deftest find-full-document-by-object-id-when-document-does-exist (deftest find-full-document-by-object-id-when-document-does-exist
(let [collection "libraries" (let [collection "libraries"
doc-id (ObjectId.) doc-id (ObjectId.)
doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }] doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }]
(mc/insert db collection doc) (mc/insert db collection doc)
(is (= (doc (mc/find-by-id db collection doc-id)))))) (is (= (to-db-object doc) (mc/find-by-id db collection doc-id)))))
(deftest find-full-document-map-by-string-id-when-document-does-exist (deftest find-full-document-map-by-string-id-when-document-does-exist
(let [collection "libraries" (let [collection "libraries"
doc-id (mu/random-uuid) doc-id (mu/random-uuid)
doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }] doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }]
(mc/insert db collection doc) (mc/insert db collection doc)
(is (= (doc (mc/find-map-by-id db collection doc-id)))))) (is (= doc (mc/find-map-by-id db collection doc-id)))))
(deftest find-full-document-map-by-object-id-when-document-does-exist (deftest find-full-document-map-by-object-id-when-document-does-exist
(let [collection "libraries" (let [collection "libraries"
doc-id (ObjectId.) doc-id (ObjectId.)
doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }] doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }]
(mc/insert db collection doc) (mc/insert db collection doc)
(is (= (doc (mc/find-map-by-id db collection doc-id)))))) (is (= doc (mc/find-map-by-id db collection doc-id)))))
(deftest find-partial-document-by-id-when-document-does-exist (deftest find-partial-document-by-id-when-document-does-exist
(let [collection "libraries" (let [collection "libraries"
doc-id (mu/random-uuid) doc-id (mu/random-uuid)
doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }] doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }]
(mc/insert db collection doc) (mc/insert db collection doc)
(is (= ({ :language "Clojure" } (is (= (to-db-object { :_id doc-id :language "Clojure" })
(mc/find-by-id db collection doc-id [ :language ])))))) (mc/find-by-id db collection doc-id [ :language ])))))
(deftest find-partial-document-as-map-by-id-when-document-does-exist (deftest find-partial-document-as-map-by-id-when-document-does-exist

11
test/monger/test/stress_test.clj
View file

@ -1,7 +1,7 @@
(ns monger.test.stress-test (ns monger.test.stress-test
(:require [monger.core :as mg] (:require [monger.core :as mg]
[monger.collection :as mc] [monger.collection :as mc]
[monger.conversion :refer [to-db-object]] [monger.conversion :refer [to-db-object from-db-object]]
[clojure.test :refer :all]) [clojure.test :refer :all])
(:import [com.mongodb WriteConcern] (:import [com.mongodb WriteConcern]
java.util.Date)) java.util.Date))
@ -30,4 +30,11 @@
(mc/remove db collection) (mc/remove db collection)
(println "Inserting " n " documents...") (println "Inserting " n " documents...")
(time (mc/insert-batch db collection docs)) (time (mc/insert-batch db collection docs))
(is (= n (mc/count db collection))))))) (is (= n (mc/count db collection))))))
(deftest ^{:performance true} convert-large-number-of-dbojects-to-maps
(doseq [n [10 100 1000 20000 40000]]
(let [docs (map (fn [i]
(to-db-object {:title "Untitled" :created-at (Date.) :number i}))
(take n (iterate inc 1)))]
(time (doall (map (fn [x] (from-db-object x true)) docs)))))))

20
test/monger/test/updating_test.clj
View file

@ -33,9 +33,9 @@
doc { :created-at date, :data-store "MongoDB", :language "Clojure", :_id doc-id } doc { :created-at date, :data-store "MongoDB", :language "Clojure", :_id doc-id }
modified-doc { :created-at date, :data-store "MongoDB", :language "Erlang", :_id doc-id }] modified-doc { :created-at date, :data-store "MongoDB", :language "Erlang", :_id doc-id }]
(mc/insert db collection doc) (mc/insert db collection doc)
(is (= (doc (mc/find-by-id db collection doc-id)))) (is (= (to-db-object doc) (mc/find-by-id db collection doc-id)))
(mc/update db collection { :_id doc-id } { :language "Erlang" }) (mc/update db collection { :_id doc-id } { $set { :language "Erlang" } })
(is (= (modified-doc (mc/find-by-id db collection doc-id)))))) (is (= (to-db-object modified-doc) (mc/find-by-id db collection doc-id)))))
(deftest ^{:updating true} update-document-by-id-without-upsert-using-update-by-id (deftest ^{:updating true} update-document-by-id-without-upsert-using-update-by-id
(let [collection "libraries" (let [collection "libraries"
@ -44,9 +44,9 @@
doc { :created-at date, :data-store "MongoDB", :language "Clojure", :_id doc-id } doc { :created-at date, :data-store "MongoDB", :language "Clojure", :_id doc-id }
modified-doc { :created-at date, :data-store "MongoDB", :language "Erlang", :_id doc-id }] modified-doc { :created-at date, :data-store "MongoDB", :language "Erlang", :_id doc-id }]
(mc/insert db collection doc) (mc/insert db collection doc)
(is (= (doc (mc/find-by-id db collection doc-id)))) (is (= (to-db-object doc) (mc/find-by-id db collection doc-id)))
(mc/update-by-id db collection doc-id { :language "Erlang" }) (mc/update-by-id db collection doc-id { $set { :language "Erlang" } })
(is (= (modified-doc (mc/find-by-id db collection doc-id)))))) (is (= (to-db-object modified-doc) (mc/find-by-id db collection doc-id)))))
(deftest ^{:updating true} update-nested-document-fields-without-upsert-using-update-by-id (deftest ^{:updating true} update-nested-document-fields-without-upsert-using-update-by-id
(let [collection "libraries" (let [collection "libraries"
@ -55,9 +55,9 @@
doc { :created-at date :data-store "MongoDB" :language { :primary "Clojure" } :_id doc-id } doc { :created-at date :data-store "MongoDB" :language { :primary "Clojure" } :_id doc-id }
modified-doc { :created-at date :data-store "MongoDB" :language { :primary "Erlang" } :_id doc-id }] modified-doc { :created-at date :data-store "MongoDB" :language { :primary "Erlang" } :_id doc-id }]
(mc/insert db collection doc) (mc/insert db collection doc)
(is (= (doc (mc/find-by-id db collection doc-id)))) (is (= (to-db-object doc) (mc/find-by-id db collection doc-id)))
(mc/update-by-id db collection doc-id { $set { "language.primary" "Erlang" }}) (mc/update-by-id db collection doc-id { $set { "language.primary" "Erlang" }})
(is (= (modified-doc (mc/find-by-id db collection doc-id)))))) (is (= (to-db-object modified-doc) (mc/find-by-id db collection doc-id)))))
(deftest ^{:updating true} update-multiple-documents (deftest ^{:updating true} update-multiple-documents
@ -151,7 +151,7 @@
(is (= 1 (mc/count db collection))) (is (= 1 (mc/count db collection)))
(is (mr/updated-existing? (mc/update db collection { :language "Clojure" } modified-doc {:multi false :upsert true}))) (is (mr/updated-existing? (mc/update db collection { :language "Clojure" } modified-doc {:multi false :upsert true})))
(is (= 1 (mc/count db collection))) (is (= 1 (mc/count db collection)))
(is (= (modified-doc (mc/find-by-id db collection doc-id)))) (is (= (to-db-object modified-doc) (mc/find-by-id db collection doc-id)))
(mc/remove db collection))) (mc/remove db collection)))
(deftest ^{:updating true} upsert-a-document-using-upsert (deftest ^{:updating true} upsert-a-document-using-upsert
@ -165,5 +165,5 @@
(is (= 1 (mc/count db collection))) (is (= 1 (mc/count db collection)))
(is (mr/updated-existing? (mc/upsert db collection {:language "Clojure"} modified-doc {:multi false}))) (is (mr/updated-existing? (mc/upsert db collection {:language "Clojure"} modified-doc {:multi false})))
(is (= 1 (mc/count db collection))) (is (= 1 (mc/count db collection)))
(is (= (modified-doc (mc/find-by-id db collection doc-id)))) (is (= (to-db-object modified-doc) (mc/find-by-id db collection doc-id)))
(mc/remove db collection)))) (mc/remove db collection))))