From daae56208525b060424fa25d428b41fb67da1dda Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 20 Mar 2012 23:28:02 +0400 Subject: [PATCH 01/10] Add core.cache to the :dev profile --- project.clj | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/project.clj b/project.clj index 2ea5929..b146cd0 100644 --- a/project.clj +++ b/project.clj @@ -15,13 +15,14 @@ :dev {:resource-paths ["test/resources"], :dependencies [[org.mongodb/mongo-java-driver "2.7.3"] [com.novemberain/validateur "1.0.0"] - [clj-time "0.3.6" :exclusions [org.clojure/clojure]] - [codox "0.3.4" :exclusions [org.clojure/clojure]] - [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]] - [org.clojure/data.json "0.1.2" :exclusions [org.clojure/clojure]] - [clj-time "0.3.6" :exclusions [org.clojure/clojure]] - [codox "0.3.4" :exclusions [org.clojure/clojure]] - [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]]]}} + [clj-time "0.3.6" :exclusions [org.clojure/clojure]] + [codox "0.3.4" :exclusions [org.clojure/clojure]] + [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]] + [org.clojure/data.json "0.1.2" :exclusions [org.clojure/clojure]] + [clj-time "0.3.6" :exclusions [org.clojure/clojure]] + [org.clojure/core.cache "0.5.0" :exclusions [org.clojure/clojure]] + [codox "0.3.4" :exclusions [org.clojure/clojure]] + [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]]]}} :aliases { "all" ["with-profile" "dev:dev,1.4"] } :repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases", :snapshots false, From 009d4c6232399eb7c9c5d68fed3fb59c17714b0f Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 16:04:50 +0400 Subject: [PATCH 02/10] Eliminate a few reflection warnings --- test/monger/test/core.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/monger/test/core.clj b/test/monger/test/core.clj index d0a8b95..f875763 100644 --- a/test/monger/test/core.clj +++ b/test/monger/test/core.clj @@ -25,20 +25,20 @@ (deftest connect-to-mongo-via-uri-without-credentials (let [connection (monger.core/connect-via-uri! "mongodb://127.0.0.1/monger-test4")] - (is (= (-> connection .getAddress (.sameHost "127.0.0.1"))))) + (is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1"))))) ;; reconnect using regular host (helper/connect!)) (deftest connect-to-mongo-via-uri-with-valid-credentials (let [connection (monger.core/connect-via-uri! "mongodb://clojurewerkz/monger!:monger!@127.0.0.1/monger-test4")] - (is (= (-> connection .getAddress (.sameHost "127.0.0.1"))))) + (is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1"))))) ;; reconnect using regular host (helper/connect!)) (if-let [uri (System/getenv "MONGOHQ_URL")] (deftest ^{:external true} connect-to-mongo-via-uri-with-valid-credentials (let [connection (monger.core/connect-via-uri! uri)] - (is (= (-> connection .getAddress (.sameHost "127.0.0.1"))))) + (is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1"))))) ;; reconnect using regular host (helper/connect!))) From f44b98ac3d2599a514621da5c1e27ced7ad694d2 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 16:05:00 +0400 Subject: [PATCH 03/10] One more test selector --- project.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index 7674642..9031e0e 100644 --- a/project.clj +++ b/project.clj @@ -6,9 +6,10 @@ [org.mongodb/mongo-java-driver "2.7.3"] [com.novemberain/validateur "1.1.0-beta1"]] :test-selectors {:default (complement :performance) - :focus :focus - :indexing :indexing - :external :external + :focus :focus + :indexing :indexing + :external :external + :cache :cache :performance :performance :all (constantly true)} :codox {:exclude [monger.internal.pagination]} From 7ef48e9bb4c0b527f37c09652b207de5187dd6c6 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 20:56:22 +0400 Subject: [PATCH 04/10] Eliminate reflection warnings --- src/monger/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monger/core.clj b/src/monger/core.clj index be62984..1deb7d7 100644 --- a/src/monger/core.clj +++ b/src/monger/core.clj @@ -267,7 +267,7 @@ MapReduceOutput (count [^MapReduceOutput this] - (.count (.results this)))) + (.count ^Iterable (.results this)))) (defn ^DBObject get-last-error "Returns the the error (if there is one) from the previous operation on this connection. From 30a4ae4ea0f63672c50ed66e054e88a7cdc30551 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 20:56:47 +0400 Subject: [PATCH 05/10] Don't assume ids are ObjectId instances (for caches it may be different) --- src/monger/collection.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/monger/collection.clj b/src/monger/collection.clj index ce6afca..9189186 100644 --- a/src/monger/collection.clj +++ b/src/monger/collection.clj @@ -312,7 +312,7 @@ (defn ^WriteResult update-by-id "Update a document with given id" - [^String collection ^ObjectId id ^Map document & { :keys [upsert write-concern] :or { upsert false + [^String collection id ^Map document & { :keys [upsert write-concern] :or { upsert false write-concern monger.core/*mongodb-write-concern* } }] (check-not-nil! id "id must not be nil") (let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection)] @@ -367,9 +367,9 @@ (defn ^WriteResult remove-by-id "Removes a single document with given id" - ([^String collection ^ObjectId id] + ([^String collection id] (remove-by-id monger.core/*mongodb-database* collection id)) - ([^DB db ^String collection ^ObjectId id] + ([^DB db ^String collection id] (check-not-nil! id "id must not be nil") (let [^DBCollection coll (.getCollection db collection)] (.remove coll (to-db-object { :_id id }))))) From 07d44ddc1f87caedda828cf8a4d97c16579fa7ba Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 20:56:57 +0400 Subject: [PATCH 06/10] ws --- test/monger/test/fixtures.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/test/monger/test/fixtures.clj b/test/monger/test/fixtures.clj index ec1ee49..8576f4e 100644 --- a/test/monger/test/fixtures.clj +++ b/test/monger/test/fixtures.clj @@ -14,3 +14,4 @@ (defcleaner locations "locations") (defcleaner domains "domains") (defcleaner pages "pages") + From e425bc84ea5452e02f4cb4556d0894f363b88fa4 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 20:57:31 +0400 Subject: [PATCH 07/10] Support DBObject conversion for all objects that implement clojure.lang.Named, not just keywords --- src/monger/conversion.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/monger/conversion.clj b/src/monger/conversion.clj index d5176d0..25bee8b 100644 --- a/src/monger/conversion.clj +++ b/src/monger/conversion.clj @@ -23,7 +23,7 @@ (ns monger.conversion (:import [com.mongodb DBObject BasicDBObject BasicDBList DBCursor] - [clojure.lang IPersistentMap Keyword Ratio] + [clojure.lang IPersistentMap Named Ratio] [java.util List Map Date] org.bson.types.ObjectId)) @@ -43,8 +43,8 @@ (to-db-object [^Ratio input] (double input)) - Keyword - (to-db-object [^Keyword input] (.getName input)) + Named + (to-db-object [^Named input] (.getName input)) IPersistentMap (to-db-object [^IPersistentMap input] From 28831c61dac7a5932049e20ffb963fd2a0d5ccbe Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 20:58:01 +0400 Subject: [PATCH 08/10] Initial monger.cache version, implements clojure.core.cache/CacheProtocol only --- src/monger/cache.clj | 52 ++++++++++++++++ test/monger/test/cache.clj | 121 +++++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 src/monger/cache.clj create mode 100644 test/monger/test/cache.clj diff --git a/src/monger/cache.clj b/src/monger/cache.clj new file mode 100644 index 0000000..3989b68 --- /dev/null +++ b/src/monger/cache.clj @@ -0,0 +1,52 @@ +(ns ^{:doc "clojure.core.cache implementation(s) on top of MongoDB." + :author "Michael S. Klishin"} + monger.cache + (:require [monger.collection :as mc] + [clojure.core.cache :as cache]) + (:use monger.conversion) + (:import [clojure.core.cache CacheProtocol])) + +;; +;; Implementation +;; + +(def ^{:const true} + default-cache-collection "cache_entries") + +;; +;; API +;; + +(defrecord BasicMongerCache [collection]) + +(extend-protocol cache/CacheProtocol + BasicMongerCache + (lookup [c k] + (:value (mc/find-map-by-id (:collection c) k))) + #_ (lookup [c k not-found] + (if-let [doc (mc/find-map-by-id (:collection c) k)] + (:value doc) + not-found)) + (has? [c k] + (mc/any? (get c :collection) {:_id k})) + (hit [this k] + this) + (miss [c k v] + (mc/insert (get c :collection) {:_id k :value v}) + c) + (evict [c k] + (mc/remove-by-id (get c :collection) k) + c) + (seed [c m] + (mc/insert-batch (get c :collection) (map (fn [[k v]] + {:_id k :value v}) m)) + c)) + + +(defn basic-monger-cache-factory + ([] + (BasicMongerCache. default-cache-collection)) + ([collection] + (BasicMongerCache. collection)) + ([collection base] + (cache/seed (BasicMongerCache. collection) base))) diff --git a/test/monger/test/cache.clj b/test/monger/test/cache.clj new file mode 100644 index 0000000..b39939d --- /dev/null +++ b/test/monger/test/cache.clj @@ -0,0 +1,121 @@ +(ns monger.test.cache + (:require [monger.test.helper :as helper] + [monger.collection :as mc]) + (:use clojure.core.cache clojure.test monger.cache) + (:import [clojure.core.cache BasicCache FIFOCache LRUCache TTLCache])) + +;; +;; Playground/Tests. These were necessary because clojure.core.cache has +;; little documentation, incomplete test suite and +;; slightly non-standard (although necessary to support all those cache variations) +;; cache operations protocol. +;; +;; This is by no means clear or complete either but it did the job of helping me +;; explore the API. + +(deftest ^{:cache true} + test-has?-with-basic-cache + (testing "that has? returns false for misses" + (let [c (BasicCache. {})] + (are [v] (is (false? (has? c v))) + :missing-key + "missing-key" + (gensym "missing-key")))) + (testing "that has? returns true for hits" + (let [c (BasicCache. {:skey "Value" :lkey (Long/valueOf 10000) "kkey" :keyword})] + (are [v] (is (has? c v)) + :skey + :lkey + "kkey")))) + + +(deftest ^{:cache true} + test-lookup-with-basic-cache + (testing "that lookup returns nil for misses" + (let [c (BasicCache. {})] + (are [v] (is (nil? (lookup c v))) + :missing-key + "missing-key" + (gensym "missing-key")))) + (testing "that lookup returns cached values for hits" + (let [l (Long/valueOf 10000) + c (BasicCache. {:skey "Value" :lkey l "kkey" :keyword})] + (are [v k] (is (= v (lookup c k))) + "Value" :skey + l :lkey + :keyword "kkey")))) + +(deftest ^{:cache true} + test-evict-with-basic-cache + (testing "that evict has no effect for keys that do not exist" + (let [c (atom (BasicCache. {:a 1 :b 2}))] + (swap! c evict :missing-key) + (is (has? @c :a)) + (is (has? @c :b)))) + (testing "that evict removes keys that did exist" + (let [c (atom (BasicCache. {:skey "Value" "kkey" :keyword}))] + (is (has? @c :skey)) + (is (= "Value" (lookup @c :skey))) + (swap! c evict :skey) + (is (not (has? @c :skey))) + (is (= nil (lookup @c :skey))) + (is (has? @c "kkey")) + (is (= :keyword (lookup @c "kkey")))))) + +(deftest ^{:cache true} + test-seed-with-basic-cache + (testing "that seed returns a new value" + (let [c (atom (BasicCache. {}))] + (swap! c seed {:a 1 :b "b" "c" :d}) + (are [k v] (do + (is (has? @c k)) + (is (= v (lookup @c k)))) + :a 1 + :b "b" + "c" :d)))) + + +;; +;; Tests +;; + +(helper/connect!) + +(use-fixtures :each (fn [f] + (mc/remove "basic_monger_cache_entries") + (f) + (mc/remove "basic_monger_cache_entries"))) + + +(deftest ^{:cache true} + test-has?-with-basic-monger-cache + (testing "that has? returns false for misses" + (let [coll "basic_monger_cache_entries" + c (basic-monger-cache-factory coll)] + )) + (testing "that has? returns true for hits" + (let [coll "basic_monger_cache_entries" + c (basic-monger-cache-factory coll {"a" 1 "b" "cache" "c" 3/4})] + (is (has? c "a")) + (is (has? c "b")) + (is (has? c "c")) + (is (not (has? c "d")))))) + + +(deftest ^{:cache true} + test-lookup-with-basic-moger-cache + (testing "that lookup returns nil for misses" + (let [coll "basic_monger_cache_entries" + c (basic-monger-cache-factory coll)] + (are [v] (is (nil? (lookup c v))) + :missing-key + "missing-key" + (gensym "missing-key")))) + (testing "that lookup returns cached values for hits" + (let [l (Long/valueOf 10000) + coll "basic_monger_cache_entries" + c (basic-monger-cache-factory coll {:skey "Value" :lkey l "kkey" :keyword})] + (are [v k] (is (= v (lookup c k))) + "Value" :skey + l :lkey + "keyword" "kkey")))) From 90171ac2d11c0bd879254d85c59bd1072eaa9c28 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 20:59:20 +0400 Subject: [PATCH 09/10] Two faster paths for strings and dates --- src/monger/conversion.clj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/monger/conversion.clj b/src/monger/conversion.clj index 25bee8b..6699abb 100644 --- a/src/monger/conversion.clj +++ b/src/monger/conversion.clj @@ -39,6 +39,14 @@ (to-db-object [input] input) + String + (to-db-object [^String input] + input) + + java.util.Date + (to-db-object [^java.util.Date input] + input) + Ratio (to-db-object [^Ratio input] (double input)) From b3a592bb3b388cd9f865ec3b18ee818299276cb0 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 21:01:18 +0400 Subject: [PATCH 10/10] Updated bechmarks --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index fd59e41..f15d249 100644 --- a/README.md +++ b/README.md @@ -160,14 +160,14 @@ are some numbers on a MacBook Pro from fall 2010 with Core i7 and an Intel SSD d ``` Testing monger.test.stress Inserting 1000 documents... -"Elapsed time: 38.317 msecs" -Inserting 10,000 documents... -"Elapsed time: 263.827 msecs" -Inserting 100,000 documents... -"Elapsed time: 1679.828 msecs" +"Elapsed time: 25.699 msecs" +Inserting 10000 documents... +"Elapsed time: 135.069 msecs" +Inserting 100000 documents... +"Elapsed time: 515.969 msecs" ``` -With the `SAFE` write concern, it takes roughly 1.7 second to insert 100,000 documents. +With the `SAFE` write concern, it takes roughly 0.5 second to insert 100,000 documents with Clojure 1.3.0. ## Regular Finders