Merge branch 'core-cache-integration'
clojure.core.cache/CacheProtocol implementation on top of MongoDB
This commit is contained in:
Michael S. Klishin
commit
42e3640151
9 changed files with 204 additions and 20 deletions
12
README.md
12
README.md
|
|
@ -160,14 +160,14 @@ are some numbers on a MacBook Pro from fall 2010 with Core i7 and an Intel SSD d
|
|||
```
|
||||
Testing monger.test.stress
|
||||
Inserting 1000 documents...
|
||||
"Elapsed time: 38.317 msecs"
|
||||
Inserting 10,000 documents...
|
||||
"Elapsed time: 263.827 msecs"
|
||||
Inserting 100,000 documents...
|
||||
"Elapsed time: 1679.828 msecs"
|
||||
"Elapsed time: 25.699 msecs"
|
||||
Inserting 10000 documents...
|
||||
"Elapsed time: 135.069 msecs"
|
||||
Inserting 100000 documents...
|
||||
"Elapsed time: 515.969 msecs"
|
||||
```
|
||||
|
||||
With the `SAFE` write concern, it takes roughly 1.7 second to insert 100,000 documents.
|
||||
With the `SAFE` write concern, it takes roughly 0.5 second to insert 100,000 documents with Clojure 1.3.0.
|
||||
|
||||
|
||||
## Regular Finders
|
||||
|
|
|
|||
10
project.clj
10
project.clj
|
|
@ -6,9 +6,10 @@
|
|||
[org.mongodb/mongo-java-driver "2.7.3"]
|
||||
[com.novemberain/validateur "1.1.0-beta1"]]
|
||||
:test-selectors {:default (complement :performance)
|
||||
:focus :focus
|
||||
:indexing :indexing
|
||||
:external :external
|
||||
:focus :focus
|
||||
:indexing :indexing
|
||||
:external :external
|
||||
:cache :cache
|
||||
:performance :performance
|
||||
:all (constantly true)}
|
||||
:codox {:exclude [monger.internal.pagination]}
|
||||
|
|
@ -21,7 +22,8 @@
|
|||
:dependencies [[clj-time "0.3.6" :exclusions [org.clojure/clojure]]
|
||||
[codox "0.3.4" :exclusions [org.clojure/clojure]]
|
||||
[org.clojure/data.json "0.1.2" :exclusions [org.clojure/clojure]]
|
||||
[org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]]]}}
|
||||
[org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]]
|
||||
[org.clojure/core.cache "0.5.0" :exclusions [org.clojure/clojure]]]}}
|
||||
:aliases { "all" ["with-profile" "dev:dev,1.4"] }
|
||||
:repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases"
|
||||
:snapshots false
|
||||
|
|
|
|||
52
src/monger/cache.clj
Normal file
52
src/monger/cache.clj
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
(ns ^{:doc "clojure.core.cache implementation(s) on top of MongoDB."
|
||||
:author "Michael S. Klishin"}
|
||||
monger.cache
|
||||
(:require [monger.collection :as mc]
|
||||
[clojure.core.cache :as cache])
|
||||
(:use monger.conversion)
|
||||
(:import [clojure.core.cache CacheProtocol]))
|
||||
|
||||
;;
|
||||
;; Implementation
|
||||
;;
|
||||
|
||||
(def ^{:const true}
|
||||
default-cache-collection "cache_entries")
|
||||
|
||||
;;
|
||||
;; API
|
||||
;;
|
||||
|
||||
(defrecord BasicMongerCache [collection])
|
||||
|
||||
(extend-protocol cache/CacheProtocol
|
||||
BasicMongerCache
|
||||
(lookup [c k]
|
||||
(:value (mc/find-map-by-id (:collection c) k)))
|
||||
#_ (lookup [c k not-found]
|
||||
(if-let [doc (mc/find-map-by-id (:collection c) k)]
|
||||
(:value doc)
|
||||
not-found))
|
||||
(has? [c k]
|
||||
(mc/any? (get c :collection) {:_id k}))
|
||||
(hit [this k]
|
||||
this)
|
||||
(miss [c k v]
|
||||
(mc/insert (get c :collection) {:_id k :value v})
|
||||
c)
|
||||
(evict [c k]
|
||||
(mc/remove-by-id (get c :collection) k)
|
||||
c)
|
||||
(seed [c m]
|
||||
(mc/insert-batch (get c :collection) (map (fn [[k v]]
|
||||
{:_id k :value v}) m))
|
||||
c))
|
||||
|
||||
|
||||
(defn basic-monger-cache-factory
|
||||
([]
|
||||
(BasicMongerCache. default-cache-collection))
|
||||
([collection]
|
||||
(BasicMongerCache. collection))
|
||||
([collection base]
|
||||
(cache/seed (BasicMongerCache. collection) base)))
|
||||
|
|
@ -312,7 +312,7 @@
|
|||
|
||||
(defn ^WriteResult update-by-id
|
||||
"Update a document with given id"
|
||||
[^String collection ^ObjectId id ^Map document & { :keys [upsert write-concern] :or { upsert false
|
||||
[^String collection id ^Map document & { :keys [upsert write-concern] :or { upsert false
|
||||
write-concern monger.core/*mongodb-write-concern* } }]
|
||||
(check-not-nil! id "id must not be nil")
|
||||
(let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection)]
|
||||
|
|
@ -367,9 +367,9 @@
|
|||
|
||||
(defn ^WriteResult remove-by-id
|
||||
"Removes a single document with given id"
|
||||
([^String collection ^ObjectId id]
|
||||
([^String collection id]
|
||||
(remove-by-id monger.core/*mongodb-database* collection id))
|
||||
([^DB db ^String collection ^ObjectId id]
|
||||
([^DB db ^String collection id]
|
||||
(check-not-nil! id "id must not be nil")
|
||||
(let [^DBCollection coll (.getCollection db collection)]
|
||||
(.remove coll (to-db-object { :_id id })))))
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@
|
|||
|
||||
(ns monger.conversion
|
||||
(:import [com.mongodb DBObject BasicDBObject BasicDBList DBCursor]
|
||||
[clojure.lang IPersistentMap Keyword Ratio]
|
||||
[clojure.lang IPersistentMap Named Ratio]
|
||||
[java.util List Map Date]
|
||||
org.bson.types.ObjectId))
|
||||
|
||||
|
|
@ -39,12 +39,20 @@
|
|||
(to-db-object [input]
|
||||
input)
|
||||
|
||||
String
|
||||
(to-db-object [^String input]
|
||||
input)
|
||||
|
||||
java.util.Date
|
||||
(to-db-object [^java.util.Date input]
|
||||
input)
|
||||
|
||||
Ratio
|
||||
(to-db-object [^Ratio input]
|
||||
(double input))
|
||||
|
||||
Keyword
|
||||
(to-db-object [^Keyword input] (.getName input))
|
||||
Named
|
||||
(to-db-object [^Named input] (.getName input))
|
||||
|
||||
IPersistentMap
|
||||
(to-db-object [^IPersistentMap input]
|
||||
|
|
|
|||
|
|
@ -267,7 +267,7 @@
|
|||
|
||||
MapReduceOutput
|
||||
(count [^MapReduceOutput this]
|
||||
(.count (.results this))))
|
||||
(.count ^Iterable (.results this))))
|
||||
|
||||
(defn ^DBObject get-last-error
|
||||
"Returns the the error (if there is one) from the previous operation on this connection.
|
||||
|
|
|
|||
121
test/monger/test/cache.clj
Normal file
121
test/monger/test/cache.clj
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
(ns monger.test.cache
|
||||
(:require [monger.test.helper :as helper]
|
||||
[monger.collection :as mc])
|
||||
(:use clojure.core.cache clojure.test monger.cache)
|
||||
(:import [clojure.core.cache BasicCache FIFOCache LRUCache TTLCache]))
|
||||
|
||||
;;
|
||||
;; Playground/Tests. These were necessary because clojure.core.cache has
|
||||
;; little documentation, incomplete test suite and
|
||||
;; slightly non-standard (although necessary to support all those cache variations)
|
||||
;; cache operations protocol.
|
||||
;;
|
||||
;; This is by no means clear or complete either but it did the job of helping me
|
||||
;; explore the API.
|
||||
|
||||
(deftest ^{:cache true}
|
||||
test-has?-with-basic-cache
|
||||
(testing "that has? returns false for misses"
|
||||
(let [c (BasicCache. {})]
|
||||
(are [v] (is (false? (has? c v)))
|
||||
:missing-key
|
||||
"missing-key"
|
||||
(gensym "missing-key"))))
|
||||
(testing "that has? returns true for hits"
|
||||
(let [c (BasicCache. {:skey "Value" :lkey (Long/valueOf 10000) "kkey" :keyword})]
|
||||
(are [v] (is (has? c v))
|
||||
:skey
|
||||
:lkey
|
||||
"kkey"))))
|
||||
|
||||
|
||||
(deftest ^{:cache true}
|
||||
test-lookup-with-basic-cache
|
||||
(testing "that lookup returns nil for misses"
|
||||
(let [c (BasicCache. {})]
|
||||
(are [v] (is (nil? (lookup c v)))
|
||||
:missing-key
|
||||
"missing-key"
|
||||
(gensym "missing-key"))))
|
||||
(testing "that lookup returns cached values for hits"
|
||||
(let [l (Long/valueOf 10000)
|
||||
c (BasicCache. {:skey "Value" :lkey l "kkey" :keyword})]
|
||||
(are [v k] (is (= v (lookup c k)))
|
||||
"Value" :skey
|
||||
l :lkey
|
||||
:keyword "kkey"))))
|
||||
|
||||
(deftest ^{:cache true}
|
||||
test-evict-with-basic-cache
|
||||
(testing "that evict has no effect for keys that do not exist"
|
||||
(let [c (atom (BasicCache. {:a 1 :b 2}))]
|
||||
(swap! c evict :missing-key)
|
||||
(is (has? @c :a))
|
||||
(is (has? @c :b))))
|
||||
(testing "that evict removes keys that did exist"
|
||||
(let [c (atom (BasicCache. {:skey "Value" "kkey" :keyword}))]
|
||||
(is (has? @c :skey))
|
||||
(is (= "Value" (lookup @c :skey)))
|
||||
(swap! c evict :skey)
|
||||
(is (not (has? @c :skey)))
|
||||
(is (= nil (lookup @c :skey)))
|
||||
(is (has? @c "kkey"))
|
||||
(is (= :keyword (lookup @c "kkey"))))))
|
||||
|
||||
(deftest ^{:cache true}
|
||||
test-seed-with-basic-cache
|
||||
(testing "that seed returns a new value"
|
||||
(let [c (atom (BasicCache. {}))]
|
||||
(swap! c seed {:a 1 :b "b" "c" :d})
|
||||
(are [k v] (do
|
||||
(is (has? @c k))
|
||||
(is (= v (lookup @c k))))
|
||||
:a 1
|
||||
:b "b"
|
||||
"c" :d))))
|
||||
|
||||
|
||||
;;
|
||||
;; Tests
|
||||
;;
|
||||
|
||||
(helper/connect!)
|
||||
|
||||
(use-fixtures :each (fn [f]
|
||||
(mc/remove "basic_monger_cache_entries")
|
||||
(f)
|
||||
(mc/remove "basic_monger_cache_entries")))
|
||||
|
||||
|
||||
(deftest ^{:cache true}
|
||||
test-has?-with-basic-monger-cache
|
||||
(testing "that has? returns false for misses"
|
||||
(let [coll "basic_monger_cache_entries"
|
||||
c (basic-monger-cache-factory coll)]
|
||||
))
|
||||
(testing "that has? returns true for hits"
|
||||
(let [coll "basic_monger_cache_entries"
|
||||
c (basic-monger-cache-factory coll {"a" 1 "b" "cache" "c" 3/4})]
|
||||
(is (has? c "a"))
|
||||
(is (has? c "b"))
|
||||
(is (has? c "c"))
|
||||
(is (not (has? c "d"))))))
|
||||
|
||||
|
||||
(deftest ^{:cache true}
|
||||
test-lookup-with-basic-moger-cache
|
||||
(testing "that lookup returns nil for misses"
|
||||
(let [coll "basic_monger_cache_entries"
|
||||
c (basic-monger-cache-factory coll)]
|
||||
(are [v] (is (nil? (lookup c v)))
|
||||
:missing-key
|
||||
"missing-key"
|
||||
(gensym "missing-key"))))
|
||||
(testing "that lookup returns cached values for hits"
|
||||
(let [l (Long/valueOf 10000)
|
||||
coll "basic_monger_cache_entries"
|
||||
c (basic-monger-cache-factory coll {:skey "Value" :lkey l "kkey" :keyword})]
|
||||
(are [v k] (is (= v (lookup c k)))
|
||||
"Value" :skey
|
||||
l :lkey
|
||||
"keyword" "kkey"))))
|
||||
|
|
@ -25,20 +25,20 @@
|
|||
|
||||
(deftest connect-to-mongo-via-uri-without-credentials
|
||||
(let [connection (monger.core/connect-via-uri! "mongodb://127.0.0.1/monger-test4")]
|
||||
(is (= (-> connection .getAddress (.sameHost "127.0.0.1")))))
|
||||
(is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1")))))
|
||||
;; reconnect using regular host
|
||||
(helper/connect!))
|
||||
|
||||
(deftest connect-to-mongo-via-uri-with-valid-credentials
|
||||
(let [connection (monger.core/connect-via-uri! "mongodb://clojurewerkz/monger!:monger!@127.0.0.1/monger-test4")]
|
||||
(is (= (-> connection .getAddress (.sameHost "127.0.0.1")))))
|
||||
(is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1")))))
|
||||
;; reconnect using regular host
|
||||
(helper/connect!))
|
||||
|
||||
(if-let [uri (System/getenv "MONGOHQ_URL")]
|
||||
(deftest ^{:external true} connect-to-mongo-via-uri-with-valid-credentials
|
||||
(let [connection (monger.core/connect-via-uri! uri)]
|
||||
(is (= (-> connection .getAddress (.sameHost "127.0.0.1")))))
|
||||
(is (= (-> connection .getAddress ^InetAddress (.sameHost "127.0.0.1")))))
|
||||
;; reconnect using regular host
|
||||
(helper/connect!)))
|
||||
|
||||
|
|
|
|||
|
|
@ -14,3 +14,4 @@
|
|||
(defcleaner locations "locations")
|
||||
(defcleaner domains "domains")
|
||||
(defcleaner pages "pages")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue