Merge branch 'core-cache-integration'

clojure.core.cache/CacheProtocol implementation on top of MongoDB

README.md

@@ -160,14 +160,14 @@ are some numbers on a MacBook Pro from fall 2010 with Core i7 and an Intel SSD d
 ```
 Testing monger.test.stress
 Inserting  1000  documents...
-"Elapsed time: 38.317 msecs"
+"Elapsed time: 25.699 msecs"
-Inserting  10,000  documents...
+Inserting  10000  documents...
-"Elapsed time: 263.827 msecs"
+"Elapsed time: 135.069 msecs"
-Inserting  100,000  documents...
+Inserting  100000  documents...
-"Elapsed time: 1679.828 msecs"
+"Elapsed time: 515.969 msecs"
 ```
 
-With the `SAFE` write concern, it takes roughly 1.7 second to insert 100,000 documents.
+With the `SAFE` write concern, it takes roughly 0.5 second to insert 100,000 documents with Clojure 1.3.0.
 
 
 ## Regular Finders

project.clj

@@ -6,9 +6,10 @@
                  [org.mongodb/mongo-java-driver "2.7.3"]
                  [com.novemberain/validateur "1.1.0-beta1"]]
   :test-selectors {:default (complement :performance)
-                   :focus :focus
+                   :focus       :focus
-                   :indexing :indexing
+                   :indexing    :indexing
-                   :external :external
+                   :external    :external
+                   :cache       :cache
                    :performance :performance
                    :all (constantly true)}
   :codox {:exclude [monger.internal.pagination]}
@@ -21,7 +22,8 @@
                    :dependencies  [[clj-time "0.3.6"              :exclusions [org.clojure/clojure]]
                                    [codox "0.3.4"                 :exclusions [org.clojure/clojure]]
                                    [org.clojure/data.json "0.1.2" :exclusions [org.clojure/clojure]]
-                                   [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]]]}}
+                                   [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]]
+                                   [org.clojure/core.cache "0.5.0" :exclusions [org.clojure/clojure]]]}}
   :aliases { "all" ["with-profile" "dev:dev,1.4"] }
   :repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases"
                              :snapshots false

src/monger/cache.clj

@@ -0,0 +1,52 @@
+(ns ^{:doc "clojure.core.cache implementation(s) on top of MongoDB."
+      :author "Michael S. Klishin"}
+  monger.cache
+  (:require [monger.collection  :as mc]
+            [clojure.core.cache :as cache])
+  (:use monger.conversion)
+  (:import [clojure.core.cache CacheProtocol]))
+
+;;
+;; Implementation
+;;
+
+(def ^{:const true}
+  default-cache-collection "cache_entries")
+
+;;
+;; API
+;;
+
+(defrecord BasicMongerCache [collection])
+
+(extend-protocol cache/CacheProtocol
+  BasicMongerCache
+  (lookup [c k]
+    (:value (mc/find-map-by-id (:collection c) k)))
+  #_ (lookup [c k not-found]
+    (if-let [doc (mc/find-map-by-id (:collection c) k)]
+      (:value doc)
+      not-found))
+  (has? [c k]
+    (mc/any? (get c :collection) {:_id k}))
+  (hit [this k]
+    this)
+  (miss [c k v]
+    (mc/insert (get c :collection) {:_id k :value v})
+    c)
+  (evict [c k]
+    (mc/remove-by-id (get c :collection) k)
+    c)
+  (seed [c m]
+    (mc/insert-batch (get c :collection) (map (fn [[k v]]
+                                                {:_id k :value v}) m))
+    c))
+
+
+(defn basic-monger-cache-factory
+  ([]
+     (BasicMongerCache. default-cache-collection))
+  ([collection]
+     (BasicMongerCache. collection))
+  ([collection base]
+     (cache/seed (BasicMongerCache. collection) base)))

src/monger/collection.clj

@@ -312,7 +312,7 @@
 
 (defn ^WriteResult update-by-id
   "Update a document with given id"
-  [^String collection ^ObjectId id ^Map document & { :keys [upsert write-concern] :or { upsert false
+  [^String collection id ^Map document & { :keys [upsert write-concern] :or { upsert false
                                                                                        write-concern monger.core/*mongodb-write-concern* } }]
   (check-not-nil! id "id must not be nil")
   (let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection)]
@@ -367,9 +367,9 @@
 
 (defn ^WriteResult remove-by-id
   "Removes a single document with given id"
-  ([^String collection ^ObjectId id]
+  ([^String collection id]
      (remove-by-id monger.core/*mongodb-database* collection id))
-  ([^DB db ^String collection ^ObjectId id]
+  ([^DB db ^String collection id]
      (check-not-nil! id "id must not be nil")
      (let [^DBCollection coll (.getCollection db collection)]
        (.remove coll (to-db-object { :_id id })))))

src/monger/conversion.clj

@@ -23,7 +23,7 @@
 
 (ns monger.conversion
   (:import [com.mongodb DBObject BasicDBObject BasicDBList DBCursor]
-           [clojure.lang IPersistentMap Keyword Ratio]
+           [clojure.lang IPersistentMap Named Ratio]
            [java.util List Map Date]
            org.bson.types.ObjectId))
 
@@ -39,12 +39,20 @@
   (to-db-object [input]
     input)
 
+  String
+  (to-db-object [^String input]
+    input)
+
+  java.util.Date
+  (to-db-object [^java.util.Date input]
+    input)
+
   Ratio
   (to-db-object [^Ratio input]
     (double input))
 
-  Keyword
+  Named
-  (to-db-object [^Keyword input] (.getName input))
+  (to-db-object [^Named input] (.getName input))
 
   IPersistentMap
   (to-db-object [^IPersistentMap input]

src/monger/core.clj

@@ -267,7 +267,7 @@
 
   MapReduceOutput
   (count [^MapReduceOutput this]
-    (.count (.results this))))
+    (.count ^Iterable (.results this))))
 
 (defn ^DBObject get-last-error
   "Returns the the error (if there is one) from the previous operation on this connection.

test/monger/test/cache.clj

@@ -0,0 +1,121 @@
+(ns monger.test.cache
+  (:require [monger.test.helper :as helper]
+            [monger.collection  :as mc])
+  (:use clojure.core.cache clojure.test monger.cache)
+  (:import [clojure.core.cache BasicCache FIFOCache LRUCache TTLCache]))
+
+;;
+;; Playground/Tests. These were necessary because clojure.core.cache has
+;; little documentation, incomplete test suite and
+;; slightly non-standard (although necessary to support all those cache variations)
+;; cache operations protocol.
+;;
+;; This is by no means clear or complete either but it did the job of helping me
+;; explore the API.
+
+(deftest ^{:cache true}
+  test-has?-with-basic-cache
+  (testing "that has? returns false for misses"
+    (let [c (BasicCache. {})]
+      (are [v] (is (false? (has? c v)))
+           :missing-key
+           "missing-key"
+           (gensym "missing-key"))))
+  (testing "that has? returns true for hits"
+    (let [c (BasicCache. {:skey "Value" :lkey (Long/valueOf 10000) "kkey" :keyword})]
+      (are [v] (is (has? c v))
+           :skey
+           :lkey
+           "kkey"))))
+
+
+(deftest ^{:cache true}
+  test-lookup-with-basic-cache
+  (testing "that lookup returns nil for misses"
+    (let [c (BasicCache. {})]
+      (are [v] (is (nil? (lookup c v)))
+           :missing-key
+           "missing-key"
+           (gensym "missing-key"))))
+  (testing "that lookup returns cached values for hits"
+    (let [l (Long/valueOf 10000)
+          c (BasicCache. {:skey "Value" :lkey l "kkey" :keyword})]
+      (are [v k] (is (= v (lookup c k)))
+           "Value"   :skey
+           l         :lkey
+           :keyword  "kkey"))))
+
+(deftest ^{:cache true}
+  test-evict-with-basic-cache
+  (testing "that evict has no effect for keys that do not exist"
+    (let [c (atom (BasicCache. {:a 1 :b 2}))]
+      (swap! c evict :missing-key)
+      (is (has? @c :a))
+      (is (has? @c :b))))
+  (testing "that evict removes keys that did exist"
+    (let [c (atom (BasicCache. {:skey "Value" "kkey" :keyword}))]
+      (is (has? @c :skey))
+      (is (= "Value"  (lookup @c :skey)))
+      (swap! c evict :skey)
+      (is (not (has? @c :skey)))
+      (is (= nil  (lookup @c :skey)))
+      (is (has? @c "kkey"))
+      (is (= :keyword (lookup @c "kkey"))))))
+
+(deftest ^{:cache true}
+  test-seed-with-basic-cache
+  (testing "that seed returns a new value"
+    (let [c (atom (BasicCache. {}))]
+      (swap! c seed {:a 1 :b "b" "c" :d})
+      (are [k v] (do
+                   (is (has? @c k))
+                   (is (= v (lookup @c k))))
+           :a 1
+           :b "b"
+           "c" :d))))
+
+
+;;
+;; Tests
+;;
+
+(helper/connect!)
+
+(use-fixtures :each (fn [f]
+                      (mc/remove "basic_monger_cache_entries")
+                      (f)
+                      (mc/remove "basic_monger_cache_entries")))
+
+
+(deftest ^{:cache true}
+  test-has?-with-basic-monger-cache
+  (testing "that has? returns false for misses"
+    (let [coll "basic_monger_cache_entries"
+          c    (basic-monger-cache-factory coll)]
+      ))
+  (testing "that has? returns true for hits"
+    (let [coll "basic_monger_cache_entries"
+          c    (basic-monger-cache-factory coll {"a" 1 "b" "cache" "c" 3/4})]
+      (is (has? c "a"))
+      (is (has? c "b"))
+      (is (has? c "c"))
+      (is (not (has? c "d"))))))
+
+
+(deftest ^{:cache true}
+  test-lookup-with-basic-moger-cache
+  (testing "that lookup returns nil for misses"
+    (let [coll "basic_monger_cache_entries"
+          c    (basic-monger-cache-factory coll)]
+      (are [v] (is (nil? (lookup c v)))
+           :missing-key
+           "missing-key"
+           (gensym "missing-key"))))
+  (testing "that lookup returns cached values for hits"
+    (let [l (Long/valueOf 10000)
+          coll "basic_monger_cache_entries"
+          c    (basic-monger-cache-factory coll {:skey "Value" :lkey l "kkey" :keyword})]
+      (are [v k] (is (= v (lookup c k)))
+           "Value"   :skey
+           l         :lkey
+           "keyword" "kkey"))))

test/monger/test/core.clj

@@ -25,20 +25,20 @@
 
 (deftest connect-to-mongo-via-uri-without-credentials
   (let [connection (monger.core/connect-via-uri! "mongodb://127.0.0.1/monger-test4")]
-    (is (= (-> connection .getAddress (.sameHost "127.0.0.1")))))
+    (is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1")))))
   ;; reconnect using regular host
   (helper/connect!))
 
 (deftest connect-to-mongo-via-uri-with-valid-credentials
   (let [connection (monger.core/connect-via-uri! "mongodb://clojurewerkz/monger!:monger!@127.0.0.1/monger-test4")]
-    (is (= (-> connection .getAddress (.sameHost "127.0.0.1")))))
+    (is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1")))))
   ;; reconnect using regular host
   (helper/connect!))
 
 (if-let [uri (System/getenv "MONGOHQ_URL")]
   (deftest ^{:external true} connect-to-mongo-via-uri-with-valid-credentials
     (let [connection (monger.core/connect-via-uri! uri)]
-      (is (= (-> connection .getAddress (.sameHost "127.0.0.1")))))
+      (is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1")))))
     ;; reconnect using regular host
     (helper/connect!)))
 

test/monger/test/fixtures.clj

@@ -14,3 +14,4 @@
 (defcleaner locations "locations")
 (defcleaner domains   "domains")
 (defcleaner pages     "pages")
+