From 7570e4ce6d09f3f718c3331a4bf60f807a5c06b1 Mon Sep 17 00:00:00 2001 From: Timo Sulg Date: Sun, 15 Sep 2013 16:53:41 +0200 Subject: [PATCH 1/4] Added cursor-helpers and new find method; --- src/clojure/monger/collection.clj | 90 ++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/src/clojure/monger/collection.clj b/src/clojure/monger/collection.clj index 95a198b..0b6a157 100644 --- a/src/clojure/monger/collection.clj +++ b/src/clojure/monger/collection.clj @@ -24,7 +24,9 @@ * http://clojuremongodb.info/articles/deleting.html * http://clojuremongodb.info/articles/aggregation.html" (:refer-clojure :exclude [find remove count drop distinct empty?]) - (:import [com.mongodb Mongo DB DBCollection WriteResult DBObject WriteConcern DBCursor MapReduceCommand MapReduceCommand$OutputType] + (:import [com.mongodb Mongo DB DBCollection WriteResult DBObject WriteConcern + DBCursor MapReduceCommand MapReduceCommand$OutputType + Bytes] [java.util List Map] [clojure.lang IPersistentMap ISeq] org.bson.types.ObjectId) @@ -34,7 +36,6 @@ monger.constraints)) - ;; ;; API ;; @@ -122,6 +123,91 @@ ;; ;; monger.collection/find ;; +;;;;TODO: add query-options +(comment + (import '[com.mongodb Mongo DB DBCollection WriteResult DBObject WriteConcern + DBCursor MapReduceCommand MapReduceCommand$OutputType + Bytes] + '[java.util List Map] + '[clojure.lang IPersistentMap ISeq] + 'org.bson.types.ObjectId) + (require '[monger.core :as mongo] + '[monger.result]) + + (require '[monger.conversion :refer [from-db-object to-db-object as-field-selector]]) + + (mongo/connect!) + (mongo/set-db! (mongo/get-db "veye_dev")) + (def collection :products) + (def coll (.getCollection monger.core/*mongodb-database* (name collection))) +) + +(defn ^DBCursor make-db-cursor + ([^String collection] (make-db-cursor collection {} {})) + ([^String collection ^Map ref] (make-db-cursor collection ref {})) + ([^String collection ^Map ref fields] + (.find + (.getCollection monger.core/*mongodb-database* (name collection)) + (to-db-object ref) + (as-field-selector fields)))) + +(def cursor-options-map {:awaitdata Bytes/QUERYOPTION_AWAITDATA + :exhaust Bytes/QUERYOPTION_EXHAUST + :notimeout Bytes/QUERYOPTION_NOTIMEOUT + :oplogreplay Bytes/QUERYOPTION_OPLOGREPLAY + :partial Bytes/QUERYOPTION_PARTIAL + :slaveok Bytes/QUERYOPTION_SLAVEOK + :tailable Bytes/QUERYOPTION_TAILABLE}) + +(defn get-cursor-options + "" + [db-cur] + (into {} + (for [[opt option-mask] cursor-options-map] + [opt (< 0 (bit-and (.getOptions db-cur) option-mask))]))) + +(defn ^DBCursor apply-cursor-options [db-cur options] + "Applies cursor options and return cursor." + (doseq [[opt value] (seq options)] + (if (= true value) + (.addOption db-cur (get cursor-options-map opt 0)) + (.setOptions db-cur (bit-and-not (.getOptions db-cur) (get cursor-options-map opt 0))))) + db-cur) + +(defmulti format-cursor (fn [db-cur as] as)) + +(defmethod format-cursor :map [db-cur as] + (map #(from-db-object %1 true) db-cur)) + +(defmethod format-cursor :seq [db-cur as] + (seq db-cur)) + +(defmethod format-cursor :default [db-cur as] + db-cur) + +(defn find-all + "Queries for objects from specified collection and accepts additional parameters as keywords. + Arguments: + collection - required, name of collections as string or keyword value + Keyword arguments: + :criteria - clojure map of matching criterias, for example {:name 'Monger', :language 'Clojure'} + :fields - specify fields, valid formats [:field1 :field2], {:_id -1, :name 1} + :options - clojure map of options for cursors, for example {:notimeout true :partial false} + allows notimeout and cancels shardings for query's cursor. + :as - specifies format of response, ala unified method to replace find, find-maps, find-seq. + Valid values are :map, :seq and other values are ignored. + + Examples: + (.count (find-all :products)) + (.count (find-all :products :criteria {:name 'Monger'} :fields {:_id -1})) + (.count (find-all :product :options {:notimeout true} :as :map)) + " + [collection & {:keys [criteria fields options as] + :or {criteria {}, fields {}, options nil, as nil}}] + (let [db-cur (make-db-cursor collection criteria fields)] + (-> db-cur + (apply-cursor-options options) + (format-cursor as)))) (defn ^DBCursor find "Queries for objects in this collection. From 58b5b1ddac4eb11b668d5d1c89d3e84f2f9008d6 Mon Sep 17 00:00:00 2001 From: Timo Sulg Date: Sun, 15 Sep 2013 20:30:01 +0200 Subject: [PATCH 2/4] updated docs; additional buffertime for timeout. --- src/clojure/monger/collection.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clojure/monger/collection.clj b/src/clojure/monger/collection.clj index 0b6a157..d17e11a 100644 --- a/src/clojure/monger/collection.clj +++ b/src/clojure/monger/collection.clj @@ -200,7 +200,7 @@ Examples: (.count (find-all :products)) (.count (find-all :products :criteria {:name 'Monger'} :fields {:_id -1})) - (.count (find-all :product :options {:notimeout true} :as :map)) + (find-all :product :options {:notimeout true} :as :map) " [collection & {:keys [criteria fields options as] :or {criteria {}, fields {}, options nil, as nil}}] From 2dca9924961526760f0536677509778d964ef2c9 Mon Sep 17 00:00:00 2001 From: Timo Sulg Date: Sat, 21 Sep 2013 15:46:01 +0200 Subject: [PATCH 3/4] added cursor namespaces, cleanups; refactored helpers for DBCursor into own namespace; cleaned up previous hacks; added extra classes for add-options; added tests for cursor namespace; --- src/clojure/monger/collection.clj | 88 +----------------------- src/clojure/monger/cursor.clj | 93 +++++++++++++++++++++++++ test/monger/test/cursor_test.clj | 110 ++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 87 deletions(-) create mode 100644 src/clojure/monger/cursor.clj create mode 100644 test/monger/test/cursor_test.clj diff --git a/src/clojure/monger/collection.clj b/src/clojure/monger/collection.clj index d17e11a..0496bd0 100644 --- a/src/clojure/monger/collection.clj +++ b/src/clojure/monger/collection.clj @@ -25,8 +25,7 @@ * http://clojuremongodb.info/articles/aggregation.html" (:refer-clojure :exclude [find remove count drop distinct empty?]) (:import [com.mongodb Mongo DB DBCollection WriteResult DBObject WriteConcern - DBCursor MapReduceCommand MapReduceCommand$OutputType - Bytes] + DBCursor MapReduceCommand MapReduceCommand$OutputType] [java.util List Map] [clojure.lang IPersistentMap ISeq] org.bson.types.ObjectId) @@ -123,91 +122,6 @@ ;; ;; monger.collection/find ;; -;;;;TODO: add query-options -(comment - (import '[com.mongodb Mongo DB DBCollection WriteResult DBObject WriteConcern - DBCursor MapReduceCommand MapReduceCommand$OutputType - Bytes] - '[java.util List Map] - '[clojure.lang IPersistentMap ISeq] - 'org.bson.types.ObjectId) - (require '[monger.core :as mongo] - '[monger.result]) - - (require '[monger.conversion :refer [from-db-object to-db-object as-field-selector]]) - - (mongo/connect!) - (mongo/set-db! (mongo/get-db "veye_dev")) - (def collection :products) - (def coll (.getCollection monger.core/*mongodb-database* (name collection))) -) - -(defn ^DBCursor make-db-cursor - ([^String collection] (make-db-cursor collection {} {})) - ([^String collection ^Map ref] (make-db-cursor collection ref {})) - ([^String collection ^Map ref fields] - (.find - (.getCollection monger.core/*mongodb-database* (name collection)) - (to-db-object ref) - (as-field-selector fields)))) - -(def cursor-options-map {:awaitdata Bytes/QUERYOPTION_AWAITDATA - :exhaust Bytes/QUERYOPTION_EXHAUST - :notimeout Bytes/QUERYOPTION_NOTIMEOUT - :oplogreplay Bytes/QUERYOPTION_OPLOGREPLAY - :partial Bytes/QUERYOPTION_PARTIAL - :slaveok Bytes/QUERYOPTION_SLAVEOK - :tailable Bytes/QUERYOPTION_TAILABLE}) - -(defn get-cursor-options - "" - [db-cur] - (into {} - (for [[opt option-mask] cursor-options-map] - [opt (< 0 (bit-and (.getOptions db-cur) option-mask))]))) - -(defn ^DBCursor apply-cursor-options [db-cur options] - "Applies cursor options and return cursor." - (doseq [[opt value] (seq options)] - (if (= true value) - (.addOption db-cur (get cursor-options-map opt 0)) - (.setOptions db-cur (bit-and-not (.getOptions db-cur) (get cursor-options-map opt 0))))) - db-cur) - -(defmulti format-cursor (fn [db-cur as] as)) - -(defmethod format-cursor :map [db-cur as] - (map #(from-db-object %1 true) db-cur)) - -(defmethod format-cursor :seq [db-cur as] - (seq db-cur)) - -(defmethod format-cursor :default [db-cur as] - db-cur) - -(defn find-all - "Queries for objects from specified collection and accepts additional parameters as keywords. - Arguments: - collection - required, name of collections as string or keyword value - Keyword arguments: - :criteria - clojure map of matching criterias, for example {:name 'Monger', :language 'Clojure'} - :fields - specify fields, valid formats [:field1 :field2], {:_id -1, :name 1} - :options - clojure map of options for cursors, for example {:notimeout true :partial false} - allows notimeout and cancels shardings for query's cursor. - :as - specifies format of response, ala unified method to replace find, find-maps, find-seq. - Valid values are :map, :seq and other values are ignored. - - Examples: - (.count (find-all :products)) - (.count (find-all :products :criteria {:name 'Monger'} :fields {:_id -1})) - (find-all :product :options {:notimeout true} :as :map) - " - [collection & {:keys [criteria fields options as] - :or {criteria {}, fields {}, options nil, as nil}}] - (let [db-cur (make-db-cursor collection criteria fields)] - (-> db-cur - (apply-cursor-options options) - (format-cursor as)))) (defn ^DBCursor find "Queries for objects in this collection. diff --git a/src/clojure/monger/cursor.clj b/src/clojure/monger/cursor.clj new file mode 100644 index 0000000..35de6df --- /dev/null +++ b/src/clojure/monger/cursor.clj @@ -0,0 +1,93 @@ +(ns monger.cursor + "Helpers function for dbCursor object: creating new cursor, + CRUD functionality for cursor options + Related documentation guides: + * ... + " + (:import [com.mongodb DBCursor Bytes] + [java.util List Map] + [java.lang Integer] + [clojure.lang Keyword]) + (:require [monger.conversion :refer [to-db-object from-db-object as-field-selector]])) + +(defn ^DBCursor make-db-cursor + "initializes new db-collection." + ([^String collection] (make-db-cursor collection {} {})) + ([^String collection ^Map ref] (make-db-cursor collection ref {})) + ([^String collection ^Map ref fields] + (.find + (.getCollection monger.core/*mongodb-database* (name collection)) + (to-db-object ref) + (as-field-selector fields)))) + +(def cursor-options {:awaitdata Bytes/QUERYOPTION_AWAITDATA + ;;:exhaust Bytes/QUERYOPTION_EXHAUST - not human settable + :notimeout Bytes/QUERYOPTION_NOTIMEOUT + :oplogreplay Bytes/QUERYOPTION_OPLOGREPLAY + :partial Bytes/QUERYOPTION_PARTIAL + :slaveok Bytes/QUERYOPTION_SLAVEOK + :tailable Bytes/QUERYOPTION_TAILABLE}) + +(defn get-options + "Returns map of cursor's options with current state." + [^DBCursor db-cur] + (into {} + (for [[opt option-mask] cursor-options] + [opt (< 0 (bit-and (.getOptions db-cur) option-mask))]))) + +(defn add-option! [^DBCursor db-cur, ^String opt] + (.addOption db-cur (get cursor-options (keyword opt) 0))) + +(defn remove-option! [^DBCursor db-cur, ^String opt] + (.setOptions db-cur (bit-and-not (.getOptions db-cur) + (get cursor-options (keyword opt) 0)))) + +(defmulti add-options (fn [db-cur opts] (class opts))) + +(defmethod add-options Map [^DBCursor db-cur options] + "Applies cursor options with switch values, where true means switch on + and false removes specified options from current cursor. + example: (add-options db-cur {:notimeout true, :tailable false}) + returns cursor." + (doseq [[opt value] (seq options)] + (if (= true value) + (add-option! db-cur opt) + (remove-option! db-cur opt))) + db-cur) + +(defmethod add-options List [^DBCursor db-cur options] + "Takes list of options to add current key" + (doseq [opt (seq options)] + (add-option! db-cur opt)) + db-cur) + +(defmethod add-options Integer [^DBCursor db-cur, option] + "Takes com.mongodb.Byte value and adds it to current settings." + (.addOption db-cur option) + db-cur) + +(defmethod add-options Keyword [^DBCursor db-cur, option] + (add-option! db-cur option) + db-cur) + +(defmethod add-options :default [^DBCursor db-cur, options] + (println "add-options dont support type for options: " (class options)) + db-cur) + +(defn ^DBCursor reset-options + "Resets cursor options to default value and returns cursor" + [^DBCursor db-cur] + (.resetOptions db-cur) + db-cur) + +(defmulti format-as (fn [db-cur as] as)) + +(defmethod format-as :map [db-cur as] + (map #(from-db-object %1 true) db-cur)) + +(defmethod format-as :seq [db-cur as] + (seq db-cur)) + +(defmethod format-as :default [db-cur as] + db-cur) + diff --git a/test/monger/test/cursor_test.clj b/test/monger/test/cursor_test.clj new file mode 100644 index 0000000..eb2cc2a --- /dev/null +++ b/test/monger/test/cursor_test.clj @@ -0,0 +1,110 @@ +(set! *warn-on-reflection* true) + +(ns monger.test.cursor-test + (:import [com.mongodb DBCursor DBObject Bytes] + [java.util List Map]) + (:require [monger.test.helper :as helper]) + (:use clojure.test + monger.cursor + monger.test.fixtures)) + +(helper/connect!) + +(deftest make-db-cursor-for-collection + (is (= DBCursor + (class (make-db-cursor :docs))))) + +(deftest getting-cursor-options-value + (let [db-cur (make-db-cursor :docs) + opts (get-options db-cur)] + (is (= true (isa? (class opts) Map))) + (is (= 0 (.getOptions db-cur))) ;;test default value + (is (= false (:notimeout opts))) + (is (= false (:partial opts))) + (is (= false (:awaitdata opts))) + (is (= false (:oplogreplay opts))) + (is (= false (:slaveok opts))) + (is (= false (:tailable opts))))) + +(deftest adding-option-to-cursor + (let [db-cur (make-db-cursor :docs) + _ (add-option! db-cur :notimeout)] + (is (= (:notimeout cursor-options) + (.getOptions db-cur))) + (add-option! db-cur :tailable) + (is (= (.getOptions db-cur) + (bit-or (:notimeout cursor-options) + (:tailable cursor-options)))))) + +(deftest remove-option-from-cursor + (let [db-cur (make-db-cursor :docs)] + (add-option! db-cur :partial) + (add-option! db-cur :awaitdata) + ;; removing not-set option should not affect result + (remove-option! db-cur :notimeout) + (is (= (.getOptions db-cur) + (bit-or (:partial cursor-options) + (:awaitdata cursor-options)))) + ;; removing active option should remove correct value + (remove-option! db-cur :awaitdata) + (is (= (.getOptions db-cur) + (:partial cursor-options))))) + + +(deftest test-reset-options + (let [db-cur (make-db-cursor :docs)] + (add-option! db-cur :partial) + (is (= (.getOptions db-cur) + (:partial cursor-options))) + (is (= 0 + (int (.getOptions (reset-options db-cur))))))) + +(deftest add-options-with-hashmap + (let [db-cur (make-db-cursor :docs) + _ (add-options db-cur {:notimeout true :slaveok true}) + opts (get-options db-cur)] + (is (= true (:notimeout opts))) + (is (= true (:slaveok opts))) + (is (= false (:tailable opts))) + (is (= false (:oplogreplay opts))))) + +(deftest add-options-with-hashmap-and-remove-option + (let [db-cur (make-db-cursor :docs) + _ (add-options db-cur {:notimeout true :slaveok true}) + opts (get-options db-cur)] + (is (= true (:notimeout opts))) + (is (= true (:slaveok opts))) + ;;remove key and add another option + (add-options db-cur {:partial true :slaveok false}) + (let [opts (get-options db-cur)] + (is (= true (:notimeout opts))) + (is (= true (:partial opts))) + (is (= false (:slaveok opts))) + (is (= false (:tailable opts)))))) + +(deftest add-options-with-list + (let [db-cur (make-db-cursor :docs) + _ (add-options db-cur [:notimeout :slaveok]) + opts (get-options db-cur)] + (is (= true (:notimeout opts))) + (is (= true (:slaveok opts))) + (is (= false (:tailable opts))) + (is (= false (:oplogreplay opts))))) + +(deftest add-options-with-Bytes + (let [db-cur (make-db-cursor :docs) + _ (add-options db-cur Bytes/QUERYOPTION_NOTIMEOUT) + opts (get-options db-cur)] + (is (= true (:notimeout opts))) + (is (= false (:slaveok opts))) + (is (= false (:tailable opts))) + (is (= false (:oplogreplay opts))))) + +(deftest add-options-with-one-keyword + (let [db-cur (make-db-cursor :docs) + _ (add-options db-cur :notimeout) + opts (get-options db-cur)] + (is (= true (:notimeout opts))) + (is (= false (:slaveok opts))) + (is (= false (:tailable opts))) + (is (= false (:oplogreplay opts))))) From 7e7f3aba4c829107c6ec2a0156e978260088ee93 Mon Sep 17 00:00:00 2001 From: Timo Sulg Date: Sat, 21 Sep 2013 19:17:58 +0200 Subject: [PATCH 4/4] cleanups; updated query/option; --- src/clojure/monger/cursor.clj | 40 +++++++++++++++++++++++------------ src/clojure/monger/query.clj | 5 +++-- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/clojure/monger/cursor.clj b/src/clojure/monger/cursor.clj index 35de6df..acded82 100644 --- a/src/clojure/monger/cursor.clj +++ b/src/clojure/monger/cursor.clj @@ -1,9 +1,7 @@ (ns monger.cursor - "Helpers function for dbCursor object: creating new cursor, - CRUD functionality for cursor options - Related documentation guides: - * ... - " + "Helper-functions for dbCursor object: + * to initialize new cursor, + * for CRUD functionality of options of dbCursor" (:import [com.mongodb DBCursor Bytes] [java.util List Map] [java.lang Integer] @@ -11,7 +9,7 @@ (:require [monger.conversion :refer [to-db-object from-db-object as-field-selector]])) (defn ^DBCursor make-db-cursor - "initializes new db-collection." + "initializes new db-cursor." ([^String collection] (make-db-cursor collection {} {})) ([^String collection ^Map ref] (make-db-cursor collection ref {})) ([^String collection ^Map ref fields] @@ -43,12 +41,13 @@ (get cursor-options (keyword opt) 0)))) (defmulti add-options (fn [db-cur opts] (class opts))) - (defmethod add-options Map [^DBCursor db-cur options] - "Applies cursor options with switch values, where true means switch on - and false removes specified options from current cursor. - example: (add-options db-cur {:notimeout true, :tailable false}) - returns cursor." + "Changes options by using map of settings, which key specifies name of settings + and boolean value specifies new state of the setting. + usage: + (add-options db-cur {:notimeout true, :tailable false}) + returns: + ^DBCursor object." (doseq [[opt value] (seq options)] (if (= true value) (add-option! db-cur opt) @@ -56,22 +55,35 @@ db-cur) (defmethod add-options List [^DBCursor db-cur options] - "Takes list of options to add current key" + "Takes list of options and activates these options + usage: + (add-options db-cur [:notimeout :tailable]) + returns: + ^DBCursor object" (doseq [opt (seq options)] (add-option! db-cur opt)) db-cur) (defmethod add-options Integer [^DBCursor db-cur, option] - "Takes com.mongodb.Byte value and adds it to current settings." + "Takes com.mongodb.Byte value and adds it to current settings. + usage: + (add-options db-cur com.mongodb.Bytes/QUERYOPTION_NOTIMEOUT) + returns: + ^DBCursor object" (.addOption db-cur option) db-cur) (defmethod add-options Keyword [^DBCursor db-cur, option] + "Takes just one keyword as name of settings and applies it to the db-cursor. + usage: + (add-options db-cur :notimeout) + returns: + ^DBCursor object" (add-option! db-cur option) db-cur) (defmethod add-options :default [^DBCursor db-cur, options] - (println "add-options dont support type for options: " (class options)) + "Using add-options with not supported type of options just passes unchanged cursor" db-cur) (defn ^DBCursor reset-options diff --git a/src/clojure/monger/query.clj b/src/clojure/monger/query.clj index 99233a8..bae17bf 100644 --- a/src/clojure/monger/query.clj +++ b/src/clojure/monger/query.clj @@ -14,7 +14,8 @@ monger.query (:refer-clojure :exclude [select find sort]) (:require [monger.core] - [monger.internal pagination]) + [monger.internal pagination] + [monger.cursor :as cursor :refer [add-options]]) (:import [com.mongodb DB DBCollection DBObject DBCursor ReadPreference] [java.util List]) (:use [monger conversion operators])) @@ -75,7 +76,7 @@ (when read-preference (.setReadPreference cursor read-preference)) (when options - (.setOptions cursor options)) + (add-options cursor options)) (map (fn [x] (from-db-object x keywordize-fields)) cursor)))