From 1db1e04b0a2933b7b0ed116981e286df4b6d6e02 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 21 Mar 2012 19:36:48 +0400 Subject: [PATCH 01/26] monger.testing => monger.testkit If nothing else, this will save everyone some headaches trying to figure out conflicts between testing the namespace alias and clojure.test/testing (the function) --- src/monger/{testing.clj => testkit.clj} | 2 +- test/monger/test/factory_dsl.clj | 2 +- test/monger/test/fixtures.clj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/monger/{testing.clj => testkit.clj} (99%) diff --git a/src/monger/testing.clj b/src/monger/testkit.clj similarity index 99% rename from src/monger/testing.clj rename to src/monger/testkit.clj index 89c9aa0..64cebad 100644 --- a/src/monger/testing.clj +++ b/src/monger/testkit.clj @@ -7,7 +7,7 @@ ;; the terms of this license. ;; You must not remove this notice, or any other, from this software. -(ns monger.testing +(ns monger.testkit (:require [monger.collection :as mc] [monger.result :as mr]) (:use [monger.internal.fn :only (expand-all expand-all-with) :as fntools]) diff --git a/test/monger/test/factory_dsl.clj b/test/monger/test/factory_dsl.clj index 08f6364..cf6c577 100644 --- a/test/monger/test/factory_dsl.clj +++ b/test/monger/test/factory_dsl.clj @@ -1,6 +1,6 @@ (ns monger.test.factory-dsl (:use [clojure.test] - [monger testing joda-time] + [monger testkit joda-time] [monger.test.fixtures] [clj-time.core :only [days ago weeks now]]) (:require [monger.collection :as mc] diff --git a/test/monger/test/fixtures.clj b/test/monger/test/fixtures.clj index 3d3a79a..ec1ee49 100644 --- a/test/monger/test/fixtures.clj +++ b/test/monger/test/fixtures.clj @@ -1,6 +1,6 @@ (ns monger.test.fixtures (:require [monger.collection :as mgcol]) - (:use [monger.testing])) + (:use monger.testkit)) ;; ;; fixture functions From 60a14e62e73909ba5c1fbc5d64017ac10758f317 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Thu, 22 Mar 2012 01:28:21 +0400 Subject: [PATCH 02/26] One more test to serve as example --- test/monger/test/updating.clj | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/monger/test/updating.clj b/test/monger/test/updating.clj index df86048..475e498 100644 --- a/test/monger/test/updating.clj +++ b/test/monger/test/updating.clj @@ -44,6 +44,17 @@ (mgcol/update-by-id collection doc-id { :language "Erlang" }) (is (= (modified-doc (mgcol/find-by-id collection doc-id)))))) +(deftest update-nested-document-fields-without-upsert-using-update-by-id + (let [collection "libraries" + doc-id (ObjectId.) + date (Date.) + doc { :created-at date :data-store "MongoDB" :language { :primary "Clojure" } :_id doc-id } + modified-doc { :created-at date :data-store "MongoDB" :language { :primary "Erlang" } :_id doc-id }] + (mgcol/insert collection doc) + (is (= (doc (mgcol/find-by-id collection doc-id)))) + (mgcol/update-by-id collection doc-id { $set { "language.primary" "Erlang" }}) + (is (= (modified-doc (mgcol/find-by-id collection doc-id)))))) + (deftest update-multiple-documents (let [collection "libraries"] From 59b32e119c1053f2c46328a66de6cce2fb38eee6 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Mon, 2 Apr 2012 12:13:05 +0400 Subject: [PATCH 03/26] Test against Clojure 1.4.0-beta6 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2ea5929..c01d13b 100644 --- a/project.clj +++ b/project.clj @@ -11,7 +11,7 @@ :archive "https://groups.google.com/group/clojure-monger", :post "clojure-monger@googlegroups.com"} :profiles {:1.4 {:resource-paths ["test/resources"], - :dependencies [[org.clojure/clojure "1.4.0-beta5"]]}, + :dependencies [[org.clojure/clojure "1.4.0-beta6"]]}, :dev {:resource-paths ["test/resources"], :dependencies [[org.mongodb/mongo-java-driver "2.7.3"] [com.novemberain/validateur "1.0.0"] From f0d190cff7ed8819d46138d49442517607d73011 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Mon, 2 Apr 2012 12:14:24 +0400 Subject: [PATCH 04/26] Support conversion of Clojure ratios to MongoDB data types We go with doubles because it is the only realistic solution that is interoperable with all other technologies. Plus, Clojure ratios are just lazily evaluated doubles anyway. --- src/monger/conversion.clj | 6 +++++- test/monger/test/conversion.clj | 5 +++++ test/monger/test/inserting.clj | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/monger/conversion.clj b/src/monger/conversion.clj index 23ca31b..76f41bb 100644 --- a/src/monger/conversion.clj +++ b/src/monger/conversion.clj @@ -23,7 +23,7 @@ (ns monger.conversion (:import [com.mongodb DBObject BasicDBObject BasicDBList DBCursor] - [clojure.lang IPersistentMap Keyword] + [clojure.lang IPersistentMap Keyword Ratio] [java.util List Map Date] [org.bson.types ObjectId])) @@ -39,6 +39,10 @@ (to-db-object [input] input) + Ratio + (to-db-object [^Ratio input] + (double input)) + Keyword (to-db-object [^Keyword input] (.getName input)) diff --git a/test/monger/test/conversion.clj b/test/monger/test/conversion.clj index 2d15643..dfa960e 100644 --- a/test/monger/test/conversion.clj +++ b/test/monger/test/conversion.clj @@ -26,6 +26,11 @@ output (cnv/to-db-object input)] (is (= input output)))) +(deftest convert-rationale-to-dbobject + (let [input 11/2 + output (cnv/to-db-object input)] + (is (= 5.5 output)))) + (deftest convert-string-to-dbobject (let [input "MongoDB" output (cnv/to-db-object input)] diff --git a/test/monger/test/inserting.clj b/test/monger/test/inserting.clj index 366e97e..7924451 100644 --- a/test/monger/test/inserting.clj +++ b/test/monger/test/inserting.clj @@ -55,6 +55,13 @@ result (mgcol/insert "people" doc)] (is (= id (monger.util/get-id doc))))) +(deftest insert-a-document-with-clojure-ratio-in-it + (let [collection "widgets" + id (ObjectId.) + doc { :ratio 11/2 "_id" id } + result (mgcol/insert "widgets" doc)] + (is (= 5.5 (:ratio (mgcol/find-map-by-id collection id)))))) + ;; From 576f264e974fc0f6ae2edd5b0cff388d72ef321d Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Mon, 2 Apr 2012 12:17:39 +0400 Subject: [PATCH 05/26] Update change log --- ChangeLog.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index ce145b4..df8a8e1 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,12 @@ ## Changes between 1.0.0-beta2 and 1.0.0-beta3 +### Support serialization of Clojure ratios + +Documents that contain Clojure ratios (for example, `26/5`) now can be converted to DBObject instances +and thus stored. On load, ratios will be presented as doubles: this way we ensure interoperability with +other languages and clients. + + ### Factories/fixtures DSL When working with even moderately complex data sets, fixture data quickly becomes difficult to From 0530b165f654f4e99ebdbb8d91f0815ad76bc305 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 3 Apr 2012 16:42:48 +0400 Subject: [PATCH 06/26] Add support for index options + unique indexes for monger.collection/ensure-index and monger.collection/create-index --- ChangeLog.md | 6 ++++++ project.clj | 6 +++++- src/monger/collection.clj | 17 ++++++++++++----- test/monger/test/collection.clj | 7 ++++--- test/monger/test/stress.clj | 2 +- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index df8a8e1..237b55f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,11 @@ ## Changes between 1.0.0-beta2 and 1.0.0-beta3 +### Index Options support for monger.collection/ensure-index and /create-index + +`monger.collection/ensure-index` and `/create-index` now accept index options as additional argument. +**Breaking change**: 3-arity versions of those functions now become 4-arity versions. + + ### Support serialization of Clojure ratios Documents that contain Clojure ratios (for example, `26/5`) now can be converted to DBObject instances diff --git a/project.clj b/project.clj index c01d13b..186a7b3 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,11 @@ :dependencies [[org.clojure/clojure "1.3.0"] [org.mongodb/mongo-java-driver "2.7.3"] [com.novemberain/validateur "1.0.0"]] - :test-selectors {:focus (fn [v] (:focus v))} + :test-selectors {:default (complement :performance) + :focus :focus + :indexing :indexing + :performance :performance + :all (constantly true)} :codox {:exclude [monger.internal.pagination]} :mailing-list {:name "clojure-monger", :archive "https://groups.google.com/group/clojure-monger", diff --git a/src/monger/collection.clj b/src/monger/collection.clj index be130ff..0cb26d6 100644 --- a/src/monger/collection.clj +++ b/src/monger/collection.clj @@ -388,16 +388,20 @@ EXAMPLES - ;; Will create an index on \"language\" field + ;; Will create an index on the \"language\" field (monger.collection/create-index collection { \"language\" 1 }) + (monger.collection/create-index collection { \"language\" 1 } { :unique true :name \"unique_language\" }) " ([^String collection ^Map keys] (let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection)] (.createIndex coll (to-db-object keys)))) - ([^DB db ^String collection ^Map keys] + ([^String collection ^Map keys options] + (let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection)] + (.createIndex coll (to-db-object keys) (to-db-object options)))) + ([^DB db ^String collection ^Map keys ^Map options] (let [^DBCollection coll (.getCollection db collection)] - (.createIndex coll (to-db-object keys))))) + (.createIndex coll (to-db-object keys) (to-db-object options))))) ;; @@ -416,9 +420,12 @@ ([^String collection, ^Map keys] (let [coll ^DBCollection (.getCollection monger.core/*mongodb-database* collection)] (.ensureIndex ^DBCollection coll ^DBObject (to-db-object keys)))) - ([^String collection, ^Map keys, ^String name] + ([^String collection, ^Map keys ^Map options] (let [coll ^DBCollection (.getCollection monger.core/*mongodb-database* collection)] - (.ensureIndex coll ^DBObject (to-db-object keys) ^String name)))) + (.ensureIndex ^DBCollection coll ^DBObject (to-db-object keys) (to-db-object options)))) + ([^String collection ^Map keys ^String name ^Boolean unique?] + (let [coll ^DBCollection (.getCollection monger.core/*mongodb-database* collection)] + (.ensureIndex coll ^DBObject (to-db-object keys) ^String name unique?)))) ;; diff --git a/test/monger/test/collection.clj b/test/monger/test/collection.clj index 1ac1650..5916839 100644 --- a/test/monger/test/collection.clj +++ b/test/monger/test/collection.clj @@ -75,7 +75,7 @@ ;; indexes ;; -(deftest index-operations +(deftest ^{:indexing true} index-operations (let [collection "libraries"] (mgcol/drop-indexes collection) (is (= "_id_" @@ -86,10 +86,11 @@ (:name (second (mgcol/indexes-on collection))))) (mgcol/drop-index collection "language_1") (is (nil? (second (mgcol/indexes-on collection)))) - (mgcol/ensure-index collection { "language" 1 }) + (mgcol/ensure-index collection { "language" 1 } {:unique true}) (is (= "language_1" (:name (second (mgcol/indexes-on collection))))) - (mgcol/ensure-index collection { "language" 1 }))) + (mgcol/ensure-index collection { "language" 1 }) + (mgcol/ensure-index collection { "language" 1 } { :unique true }))) ;; diff --git a/test/monger/test/stress.clj b/test/monger/test/stress.clj index e0d981b..9efd301 100644 --- a/test/monger/test/stress.clj +++ b/test/monger/test/stress.clj @@ -30,7 +30,7 @@ (monger.core/set-default-write-concern! WriteConcern/NORMAL) -(deftest insert-large-batches-of-documents-without-object-ids +(deftest ^{:performance true} insert-large-batches-of-documents-without-object-ids (doseq [n [1000 10000 100000]] (let [collection "things" docs (map (fn [i] From c899c482634eeaae7f122263e85394b57fad543d Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 3 Apr 2012 16:43:44 +0400 Subject: [PATCH 07/26] Make sure we drop indexes after this test --- test/monger/test/collection.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/monger/test/collection.clj b/test/monger/test/collection.clj index 5916839..b31faa1 100644 --- a/test/monger/test/collection.clj +++ b/test/monger/test/collection.clj @@ -90,7 +90,8 @@ (is (= "language_1" (:name (second (mgcol/indexes-on collection))))) (mgcol/ensure-index collection { "language" 1 }) - (mgcol/ensure-index collection { "language" 1 } { :unique true }))) + (mgcol/ensure-index collection { "language" 1 } { :unique true }) + (mgcol/drop-indexes collection))) ;; From 88961ddf5bfa5c3bdd20f6c4e3fd58e28be585f2 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 3 Apr 2012 16:46:39 +0400 Subject: [PATCH 08/26] Validateur 1.1.0-beta1 --- project.clj | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/project.clj b/project.clj index 186a7b3..708fe45 100644 --- a/project.clj +++ b/project.clj @@ -4,19 +4,19 @@ :license {:name "Eclipse Public License"} :dependencies [[org.clojure/clojure "1.3.0"] [org.mongodb/mongo-java-driver "2.7.3"] - [com.novemberain/validateur "1.0.0"]] + [com.novemberain/validateur "1.1.0-beta1"]] :test-selectors {:default (complement :performance) :focus :focus :indexing :indexing :performance :performance :all (constantly true)} :codox {:exclude [monger.internal.pagination]} - :mailing-list {:name "clojure-monger", - :archive "https://groups.google.com/group/clojure-monger", + :mailing-list {:name "clojure-monger" + :archive "https://groups.google.com/group/clojure-monger" :post "clojure-monger@googlegroups.com"} - :profiles {:1.4 {:resource-paths ["test/resources"], - :dependencies [[org.clojure/clojure "1.4.0-beta6"]]}, - :dev {:resource-paths ["test/resources"], + :profiles {:1.4 {:resource-paths ["test/resources"] + :dependencies [[org.clojure/clojure "1.4.0-beta6"]]} + :dev {:resource-paths ["test/resources"] :dependencies [[org.mongodb/mongo-java-driver "2.7.3"] [com.novemberain/validateur "1.0.0"] [clj-time "0.3.6" :exclusions [org.clojure/clojure]] @@ -27,7 +27,7 @@ [codox "0.3.4" :exclusions [org.clojure/clojure]] [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]]]}} :aliases { "all" ["with-profile" "dev:dev,1.4"] } - :repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases", - :snapshots false, - :releases {:checksum :fail, :update :always}}} + :repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases" + :snapshots false + :releases {:checksum :fail :update :always}}} :warn-on-reflection true) \ No newline at end of file From a68d8652e3924836d1ea21a7855c34b421388799 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 3 Apr 2012 16:47:30 +0400 Subject: [PATCH 09/26] Document Validateur upgrade in the log --- ChangeLog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index 237b55f..9495e32 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,10 @@ ## Changes between 1.0.0-beta2 and 1.0.0-beta3 +### Validateur 1.1.0-beta1 + +[Validateur](https://github.com/michaelklishin/validateur) dependency has been upgraded to 1.1.0-beta1. + + ### Index Options support for monger.collection/ensure-index and /create-index `monger.collection/ensure-index` and `/create-index` now accept index options as additional argument. From 6282f41f068da2e0838e3303326c45a68a2fc195 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 4 Apr 2012 23:08:05 +0400 Subject: [PATCH 10/26] Support field negation in queries, closes #17 --- ChangeLog.md | 14 +++++++ src/monger/collection.clj | 40 +++++++++---------- src/monger/conversion.clj | 22 ++++++++-- src/monger/query.clj | 6 +-- test/monger/test/conversion.clj | 60 +++++++++++++++++----------- test/monger/test/regular_finders.clj | 20 +++++++--- 6 files changed, 103 insertions(+), 59 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 9495e32..ac55dbd 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,19 @@ ## Changes between 1.0.0-beta2 and 1.0.0-beta3 +### Support for field negation in queries + +Previously to load only a subset of document fields with Monger, one had to specify them all. Starting +with 1.0.0-beta3, Monger supports [field negation](http://www.mongodb.org/display/DOCS/Retrieving+a+Subset+of+Fields#RetrievingaSubsetofFields-FieldNegation) feature of MongoDB: it is possible to exclude +certain fields instead. + +To do so, pass a map as field selector, with fields that should be omitted set to 0: + +``` clojure +;; will retrieve all fields except body +(monger.collection/find-one-map "documents" {:author "John"} {:body 0}) +``` + + ### Validateur 1.1.0-beta1 [Validateur](https://github.com/michaelklishin/validateur) dependency has been upgraded to 1.1.0-beta1. diff --git a/src/monger/collection.clj b/src/monger/collection.clj index 0cb26d6..ce6afca 100644 --- a/src/monger/collection.clj +++ b/src/monger/collection.clj @@ -21,10 +21,6 @@ ;; Implementation ;; -(defn- fields-to-db-object - [^List fields] - (zipmap fields (repeat 1))) - (definline check-not-nil! [ref ^String message] `(when (nil? ~ref) @@ -104,13 +100,13 @@ ([^String collection ^Map ref] (let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection)] (.find ^DBCollection coll ^DBObject (to-db-object ref)))) - ([^String collection ^Map ref ^List fields] + ([^String collection ^Map ref fields] (let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection) - map-of-fields (fields-to-db-object fields)] + map-of-fields (as-field-selector fields)] (.find ^DBCollection coll ^DBObject (to-db-object ref) ^DBObject (to-db-object map-of-fields)))) - ([^DB db ^String collection ^Map ref ^List fields] + ([^DB db ^String collection ^Map ref fields] (let [^DBCollection coll (.getCollection db collection) - map-of-fields (fields-to-db-object fields)] + map-of-fields (as-field-selector fields)] (.find ^DBCollection coll ^DBObject (to-db-object ref) ^DBObject (to-db-object map-of-fields))))) (defn ^ISeq find-maps @@ -122,9 +118,9 @@ (map (fn [x] (from-db-object x true)) (find collection))) ([^String collection ^Map ref] (map (fn [x] (from-db-object x true)) (find collection ref))) - ([^String collection ^Map ref ^List fields] + ([^String collection ^Map ref fields] (map (fn [x] (from-db-object x true)) (find collection ref fields))) - ([^DB db ^String collection ^Map ref ^List fields] + ([^DB db ^String collection ^Map ref fields] (map (fn [x] (from-db-object x true)) (find db collection ref fields)))) (defn ^ISeq find-seq @@ -133,9 +129,9 @@ (seq (find collection))) ([^String collection ^Map ref] (seq (find collection ref))) - ([^String collection ^Map ref ^List fields] + ([^String collection ^Map ref fields] (seq (find collection ref fields))) - ([^DB db ^String collection ^Map ref ^List fields] + ([^DB db ^String collection ^Map ref fields] (seq (find db collection ref fields)))) ;; @@ -157,22 +153,22 @@ ([^String collection ^Map ref] (let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection)] (.findOne ^DBCollection coll ^DBObject (to-db-object ref)))) - ([^String collection ^Map ref ^List fields] + ([^String collection ^Map ref fields] (let [^DBCollection coll (.getCollection monger.core/*mongodb-database* collection) - map-of-fields (fields-to-db-object fields)] + map-of-fields (as-field-selector fields)] (.findOne ^DBCollection coll ^DBObject (to-db-object ref) ^DBObject (to-db-object map-of-fields)))) - ([^DB db ^String collection ^Map ref ^List fields] + ([^DB db ^String collection ^Map ref fields] (let [^DBCollection coll (.getCollection db collection) - map-of-fields (fields-to-db-object fields)] + map-of-fields (as-field-selector fields)] (.findOne ^DBCollection coll ^DBObject (to-db-object ref) ^DBObject (to-db-object map-of-fields))))) (defn ^IPersistentMap find-one-as-map "Returns a single object converted to Map from this collection matching the query." ([^String collection ^Map ref] (from-db-object ^DBObject (find-one collection ref) true)) - ([^String collection ^Map ref ^List fields] + ([^String collection ^Map ref fields] (from-db-object ^DBObject (find-one collection ref fields) true)) - ([^String collection ^Map ref ^List fields keywordize] + ([^String collection ^Map ref fields keywordize] (from-db-object ^DBObject (find-one collection ref fields) keywordize))) @@ -195,10 +191,10 @@ ([^String collection id] (check-not-nil! id "id must not be nil") (find-one collection { :_id id })) - ([^String collection id ^List fields] + ([^String collection id fields] (check-not-nil! id "id must not be nil") (find-one collection { :_id id } fields)) - ([^DB db ^String collection id ^List fields] + ([^DB db ^String collection id fields] (check-not-nil! id "id must not be nil") (find-one db collection { :_id id } fields))) @@ -207,10 +203,10 @@ ([^String collection id] (check-not-nil! id "id must not be nil") (from-db-object ^DBObject (find-one-as-map collection { :_id id }) true)) - ([^String collection id ^List fields] + ([^String collection id fields] (check-not-nil! id "id must not be nil") (from-db-object ^DBObject (find-one-as-map collection { :_id id } fields) true)) - ([^String collection id ^List fields keywordize] + ([^String collection id fields keywordize] (check-not-nil! id "id must not be nil") (from-db-object ^DBObject (find-one-as-map collection { :_id id } fields) keywordize))) diff --git a/src/monger/conversion.clj b/src/monger/conversion.clj index 76f41bb..d5176d0 100644 --- a/src/monger/conversion.clj +++ b/src/monger/conversion.clj @@ -25,10 +25,10 @@ (:import [com.mongodb DBObject BasicDBObject BasicDBList DBCursor] [clojure.lang IPersistentMap Keyword Ratio] [java.util List Map Date] - [org.bson.types ObjectId])) + org.bson.types.ObjectId)) (defprotocol ConvertToDBObject - (to-db-object [input] "Converts given piece of Clojure data to BasicDBObject MongoDB Java driver uses")) + (^com.mongodb.DBObject to-db-object [input] "Converts given piece of Clojure data to BasicDBObject MongoDB Java driver uses")) (extend-protocol ConvertToDBObject nil @@ -109,7 +109,7 @@ (defprotocol ConvertToObjectId - (to-object-id [input] "Instantiates ObjectId from input unless the input itself is an ObjectId instance. In that case, returns input as is.")) + (^org.bson.types.ObjectId to-object-id [input] "Instantiates ObjectId from input unless the input itself is an ObjectId instance. In that case, returns input as is.")) (extend-protocol ConvertToObjectId String @@ -125,3 +125,19 @@ input)) + +(defprotocol FieldSelector + (^com.mongodb.DBObject as-field-selector [input] "Converts values to DBObject that can be used to specify a list of document fields (including negation support)")) + +(extend-protocol FieldSelector + DBObject + (as-field-selector [^DBObject input] + input) + + List + (as-field-selector [^List input] + (to-db-object (zipmap input (repeat 1)))) + + Object + (as-field-selector [input] + (to-db-object input))) diff --git a/src/monger/query.clj b/src/monger/query.clj index 0b7a64c..a25a68d 100644 --- a/src/monger/query.clj +++ b/src/monger/query.clj @@ -58,13 +58,9 @@ ([^DBCollection coll] (merge (empty-query) { :collection coll }))) -(defn- fields-to-db-object - [^List fields] - (to-db-object (zipmap fields (repeat 1)))) - (defn exec [{ :keys [collection query fields skip limit sort batch-size hint snapshot read-preference keywordize-fields] :or { limit 0 batch-size 256 skip 0 } }] - (let [cursor (doto ^DBCursor (.find ^DBCollection collection (to-db-object query) (fields-to-db-object fields)) + (let [cursor (doto ^DBCursor (.find ^DBCollection collection (to-db-object query) (as-field-selector fields)) (.limit limit) (.skip skip) (.sort (to-db-object sort)) diff --git a/test/monger/test/conversion.clj b/test/monger/test/conversion.clj index dfa960e..945f4f5 100644 --- a/test/monger/test/conversion.clj +++ b/test/monger/test/conversion.clj @@ -1,10 +1,9 @@ (ns monger.test.conversion - (:require [monger core collection] - [monger.conversion :as cnv]) + (:require [monger core collection]) (:import [com.mongodb DBObject BasicDBObject BasicDBList] [java.util Date Calendar List ArrayList] [org.bson.types ObjectId]) - (:use [clojure.test])) + (:use clojure.test monger.conversion)) ;; @@ -13,33 +12,33 @@ (deftest convert-nil-to-dbobject (let [input nil - output (cnv/to-db-object input)] + output (to-db-object input)] (is (nil? output)))) (deftest convert-integer-to-dbobject (let [input 1 - output (cnv/to-db-object input)] + output (to-db-object input)] (is (= input output)))) (deftest convert-float-to-dbobject (let [input 11.12 - output (cnv/to-db-object input)] + output (to-db-object input)] (is (= input output)))) (deftest convert-rationale-to-dbobject (let [input 11/2 - output (cnv/to-db-object input)] + output (to-db-object input)] (is (= 5.5 output)))) (deftest convert-string-to-dbobject (let [input "MongoDB" - output (cnv/to-db-object input)] + output (to-db-object input)] (is (= input output)))) (deftest convert-map-to-dbobject (let [input { :int 1, :string "Mongo", :float 22.23 } - output ^DBObject (cnv/to-db-object input)] + output ^DBObject (to-db-object input)] (is (= 1 (.get output "int"))) (is (= "Mongo" (.get output "string"))) (is (= 22.23 (.get output "float"))))) @@ -47,7 +46,7 @@ (deftest convert-nested-map-to-dbobject (let [input { :int 1, :string "Mongo", :float 22.23, :map { :int 10, :string "Clojure", :float 11.9, :list '(1 "a" :b), :map { :key "value" } } } - output ^DBObject (cnv/to-db-object input) + output ^DBObject (to-db-object input) inner ^DBObject (.get output "map")] (is (= 10 (.get inner "int"))) (is (= "Clojure" (.get inner "string"))) @@ -59,19 +58,19 @@ ;; to obtain _id that was generated. MK. (deftest convert-dbobject-to-dbobject (let [input (BasicDBObject.) - output (cnv/to-db-object input)] + output (to-db-object input)] (is (= input output)))) (deftest convert-java-date-to-dbobject (let [date (Date.) input { :int 1, :string "Mongo", :date date } - output ^DBObject (cnv/to-db-object input)] + output ^DBObject (to-db-object input)] (is (= date (.get output "date"))))) (deftest convert-java-calendar-instance-to-dbobject (let [date (Calendar/getInstance) input { :int 1, :string "Mongo", :date date } - output ^DBObject (cnv/to-db-object input)] + output ^DBObject (to-db-object input)] (is (= date (.get output "date"))))) @@ -82,16 +81,16 @@ ;; (deftest convert-nil-from-db-object - (is (nil? (cnv/from-db-object nil false))) - (is (nil? (cnv/from-db-object nil true)))) + (is (nil? (from-db-object nil false))) + (is (nil? (from-db-object nil true)))) (deftest convert-integer-from-dbobject - (is (= 2 (cnv/from-db-object 2 false))) - (is (= 2 (cnv/from-db-object 2 true)))) + (is (= 2 (from-db-object 2 false))) + (is (= 2 (from-db-object 2 true)))) (deftest convert-float-from-dbobject - (is (= 3.3 (cnv/from-db-object 3.3 false))) - (is (= 3.3 (cnv/from-db-object 3.3 true)))) + (is (= 3.3 (from-db-object 3.3 false))) + (is (= 3.3 (from-db-object 3.3 true)))) (deftest convert-flat-db-object-to-map-without-keywordizing (let [name "Michael" @@ -99,7 +98,7 @@ input (doto (BasicDBObject.) (.put "name" name) (.put "age" age)) - output (cnv/from-db-object input false)] + output (from-db-object input false)] (is (= (output { "name" name, "age" age }))) (is (= (output "name") name)) (is (nil? (output :name))) @@ -112,7 +111,7 @@ input (doto (BasicDBObject.) (.put "name" name) (.put "age" age)) - output (cnv/from-db-object input true)] + output (from-db-object input true)] (is (= (output { :name name, :age age }))) (is (= (output :name) name)) (is (nil? (output "name"))) @@ -131,7 +130,7 @@ input (doto (BasicDBObject.) (.put "_id" did) (.put "nested" nested)) - output (cnv/from-db-object input false)] + output (from-db-object input false)] (is (= (output "_id") did)) (is (= (-> output (get "nested") (get "int")) 101)) (is (= (-> output (get "nested") (get "list")) ["red" "green" "blue"])) @@ -145,5 +144,18 @@ (deftest test-conversion-to-object-id (let [output (ObjectId. "4efb39370364238a81020502")] - (is (= output (cnv/to-object-id "4efb39370364238a81020502"))) - (is (= output (cnv/to-object-id output))))) + (is (= output (to-object-id "4efb39370364238a81020502"))) + (is (= output (to-object-id output))))) + + +;; +;; Field selector coercion +;; + +(deftest test-field-selector-coercion + (are [i o] (is (= (from-db-object (as-field-selector i) true) o)) + [:a :b :c] {:a 1 :b 1 :c 1} + '(:a :b :c) {:a 1 :b 1 :c 1} + {:a 1 :b 1 :c 1} {:a 1 :b 1 :c 1} + {"a" 1 "b" 1 "c" 1} {:a 1 :b 1 :c 1} + {:comments 0} {:comments 0})) diff --git a/test/monger/test/regular_finders.clj b/test/monger/test/regular_finders.clj index 79f5753..f5af0eb 100644 --- a/test/monger/test/regular_finders.clj +++ b/test/monger/test/regular_finders.clj @@ -54,21 +54,31 @@ (let [collection "docs" doc-id (monger.util/random-uuid) doc { :data-store "MongoDB", :language "Clojure", :_id doc-id } - fields [:language] _ (mgcol/insert collection doc) - loaded (mgcol/find-one collection { :language "Clojure" } fields)] + loaded (mgcol/find-one collection { :language "Clojure" } [:language])] (is (nil? (.get ^DBObject loaded "data-store"))) (is (= doc-id (monger.util/get-id loaded))) (is (= "Clojure" (.get ^DBObject loaded "language"))))) +(deftest find-one-partial-document-using-field-negation-when-collection-has-matches + (let [collection "docs" + doc-id (monger.util/random-uuid) + doc { :data-store "MongoDB", :language "Clojure", :_id doc-id } + _ (mgcol/insert collection doc) + ^DBObject loaded (mgcol/find-one collection { :language "Clojure" } {:data-store 0 :_id 0})] + (is (nil? (.get loaded "data-store"))) + (is (nil? (.get loaded "_id"))) + (is (nil? (monger.util/get-id loaded))) + (is (= "Clojure" (.get loaded "language"))))) + + (deftest find-one-partial-document-as-map-when-collection-has-matches (let [collection "docs" doc-id (monger.util/random-uuid) - doc { :data-store "MongoDB", :language "Clojure", :_id doc-id } - fields [:data-store]] + doc { :data-store "MongoDB", :language "Clojure", :_id doc-id }] (mgcol/insert collection doc) - (is (= { :data-store "MongoDB", :_id doc-id } (mgcol/find-one-as-map collection { :language "Clojure" } fields))))) + (is (= { :data-store "MongoDB", :_id doc-id } (mgcol/find-one-as-map collection { :language "Clojure" } [:data-store]))))) (deftest find-one-partial-document-as-map-when-collection-has-matches-with-keywordize From c23a589620ead08cf14b1c8d45cf3d2018edca99 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 4 Apr 2012 23:23:31 +0400 Subject: [PATCH 11/26] Correct comment --- src/monger/query.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monger/query.clj b/src/monger/query.clj index a25a68d..c9224e3 100644 --- a/src/monger/query.clj +++ b/src/monger/query.clj @@ -31,7 +31,7 @@ ;; ;; Existing query fields: ;; -;; :fields - selects which fields are returned. The default is all fields. _id is always returned. +;; :fields - selects which fields are returned. The default is all fields. _id is included by default. ;; :sort - adds a sort to the query. ;; :fields - set of fields to retrieve during query execution ;; :skip - Skips the first N results. From e48462b250e72573652c2c571e1722c4a58102ad Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Fri, 6 Apr 2012 00:00:29 +0400 Subject: [PATCH 12/26] Remove extra dependencies It was a mistake back from when I was only figuring out Leiningen 2 profiles --- project.clj | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/project.clj b/project.clj index 708fe45..c334b91 100644 --- a/project.clj +++ b/project.clj @@ -17,14 +17,9 @@ :profiles {:1.4 {:resource-paths ["test/resources"] :dependencies [[org.clojure/clojure "1.4.0-beta6"]]} :dev {:resource-paths ["test/resources"] - :dependencies [[org.mongodb/mongo-java-driver "2.7.3"] - [com.novemberain/validateur "1.0.0"] - [clj-time "0.3.6" :exclusions [org.clojure/clojure]] + :dependencies [[clj-time "0.3.6" :exclusions [org.clojure/clojure]] [codox "0.3.4" :exclusions [org.clojure/clojure]] - [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]] [org.clojure/data.json "0.1.2" :exclusions [org.clojure/clojure]] - [clj-time "0.3.6" :exclusions [org.clojure/clojure]] - [codox "0.3.4" :exclusions [org.clojure/clojure]] [org.clojure/tools.cli "0.2.1" :exclusions [org.clojure/clojure]]]}} :aliases { "all" ["with-profile" "dev:dev,1.4"] } :repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases" From 876269e81a2f3cf335f5b3ca04f235b2979bbb66 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Sat, 7 Apr 2012 07:42:13 +0400 Subject: [PATCH 13/26] Use 127.0.0.1 as default host --- src/monger/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monger/core.clj b/src/monger/core.clj index 8584312..b2f0c06 100644 --- a/src/monger/core.clj +++ b/src/monger/core.clj @@ -22,7 +22,7 @@ ;; Defaults ;; -(def ^:dynamic ^String *mongodb-host* "localhost") +(def ^:dynamic ^String *mongodb-host* "127.0.0.1") (def ^:dynamic ^long *mongodb-port* 27017) (declare ^:dynamic ^Mongo *mongodb-connection*) From 05669a6778472963f4b6494dd48be3544ee8a848 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Sun, 8 Apr 2012 23:52:33 +0400 Subject: [PATCH 14/26] Correct query DSL example in the README, references #19 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ca1f13..f58c963 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ Here is what monger.query DSL feels like: (with-collection "movies" (find { :year { $lt 2010 $gte 2000 }, :revenue { $gt 20000000 } }) (fields [ :year :title :producer :cast :budget :revenue ]) - (sort-by { :revenue -1 }) + (sort { :revenue -1 }) (skip 10) (limit 20) (hint "year-by-year-revenue-idx") From fbd631b22efa4bbdf00770a99fe933b7d059c506 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 10 Apr 2012 16:12:59 +0400 Subject: [PATCH 15/26] Add a test that inserts documents with fields that are Clojure records --- test/monger/test/inserting.clj | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/monger/test/inserting.clj b/test/monger/test/inserting.clj index 7924451..6538096 100644 --- a/test/monger/test/inserting.clj +++ b/test/monger/test/inserting.clj @@ -63,6 +63,17 @@ (is (= 5.5 (:ratio (mgcol/find-map-by-id collection id)))))) +(defrecord Metrics + [rps eps]) + +(deftest ^:focus insert-a-document-with-clojure-record-in-it + (let [collection "widgets" + id (ObjectId.) + doc { :record (Metrics. 10 20) "_id" id } + result (mgcol/insert "widgets" doc)] + (is (= {:rps 10 :eps 20} (:record (mgcol/find-map-by-id collection id)))))) + + ;; ;; insert-batch From 52e824cdd82442e24fde0f842bd0a92a64de7bba Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 10 Apr 2012 16:15:05 +0400 Subject: [PATCH 16/26] Test against Clojure 1.4.0-beta7 --- README.md | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f58c963..8d69329 100644 --- a/README.md +++ b/README.md @@ -362,7 +362,7 @@ Neocons is part of the group of libraries known as ClojureWerkz, together with ## Development Monger uses [Leiningen 2](https://github.com/technomancy/leiningen/blob/master/doc/TUTORIAL.md). Make sure you have it installed and then run tests against -Clojure 1.3.0 and 1.4.0[-beta4] using +supported Clojure versions using lein2 all test diff --git a/project.clj b/project.clj index c334b91..44c18b9 100644 --- a/project.clj +++ b/project.clj @@ -15,7 +15,7 @@ :archive "https://groups.google.com/group/clojure-monger" :post "clojure-monger@googlegroups.com"} :profiles {:1.4 {:resource-paths ["test/resources"] - :dependencies [[org.clojure/clojure "1.4.0-beta6"]]} + :dependencies [[org.clojure/clojure "1.4.0-beta7"]]} :dev {:resource-paths ["test/resources"] :dependencies [[clj-time "0.3.6" :exclusions [org.clojure/clojure]] [codox "0.3.4" :exclusions [org.clojure/clojure]] From 8f18d30e433bace751e84572ebb031e8c4e8ec9c Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 10 Apr 2012 19:11:06 +0400 Subject: [PATCH 17/26] Extract map/reduce and indexing tests --- test/monger/test/collection.clj | 130 ++++++-------------------------- test/monger/test/indexing.clj | 35 +++++++++ test/monger/test/map_reduce.clj | 69 +++++++++++++++++ 3 files changed, 129 insertions(+), 105 deletions(-) create mode 100644 test/monger/test/indexing.clj create mode 100644 test/monger/test/map_reduce.clj diff --git a/test/monger/test/collection.clj b/test/monger/test/collection.clj index b31faa1..65e6935 100644 --- a/test/monger/test/collection.clj +++ b/test/monger/test/collection.clj @@ -2,18 +2,15 @@ (ns monger.test.collection (:import [com.mongodb WriteResult WriteConcern DBCursor DBObject CommandResult$CommandFailure MapReduceOutput MapReduceCommand MapReduceCommand$OutputType] - [org.bson.types ObjectId] - [java.util Date]) + org.bson.types.ObjectId + java.util.Date) (:require [monger core util] - [clojure stacktrace] - [monger.collection :as mgcol] - [monger.result :as mgres] - [monger.conversion :as mgcnv] - [monger.js :as js] + [monger.collection :as mgcol] + [monger.result :as mgres] [monger.test.helper :as helper]) - (:use [clojure.test] - [monger.operators] - [monger.test.fixtures])) + (:use clojure.test + monger.operators + monger.test.fixtures)) (helper/connect!) @@ -71,29 +68,6 @@ (is (nil? (mgcol/find-by-id collection oid))))) -;; -;; indexes -;; - -(deftest ^{:indexing true} index-operations - (let [collection "libraries"] - (mgcol/drop-indexes collection) - (is (= "_id_" - (:name (first (mgcol/indexes-on collection))))) - (is (nil? (second (mgcol/indexes-on collection)))) - (mgcol/create-index collection { "language" 1 }) - (is (= "language_1" - (:name (second (mgcol/indexes-on collection))))) - (mgcol/drop-index collection "language_1") - (is (nil? (second (mgcol/indexes-on collection)))) - (mgcol/ensure-index collection { "language" 1 } {:unique true}) - (is (= "language_1" - (:name (second (mgcol/indexes-on collection))))) - (mgcol/ensure-index collection { "language" 1 }) - (mgcol/ensure-index collection { "language" 1 } { :unique true }) - (mgcol/drop-indexes collection))) - - ;; ;; exists?, drop, create ;; @@ -118,65 +92,36 @@ (is (mgcol/exists? "gadgets")) (mgcol/drop "gadgets"))) - ;; -;; Map/Reduce +;; any?, empty? ;; -(let [collection "widgets" - mapper (js/load-resource "resources/mongo/js/mapfun1.js") - reducer "function(key, values) { - var result = 0; - values.forEach(function(v) { result += v }); +(deftest test-any-on-empty-collection + (let [collection "things"] + (is (not (mgcol/any? collection))))) - return result; - }" - batch [{ :state "CA" :quantity 1 :price 199.00 } - { :state "NY" :quantity 2 :price 199.00 } - { :state "NY" :quantity 1 :price 299.00 } - { :state "IL" :quantity 2 :price 11.50 } - { :state "CA" :quantity 2 :price 2.95 } - { :state "IL" :quantity 3 :price 5.50 }] - expected [{:_id "CA", :value 204.9} {:_id "IL", :value 39.5} {:_id "NY", :value 697.0}]] - (deftest basic-inline-map-reduce-example - (mgcol/remove monger.core/*mongodb-database* collection {}) - (is (mgres/ok? (mgcol/insert-batch collection batch))) - (let [output (mgcol/map-reduce collection mapper reducer nil MapReduceCommand$OutputType/INLINE {}) - results (mgcnv/from-db-object ^DBObject (.results ^MapReduceOutput output) true)] - (mgres/ok? output) - (is (= expected results)))) +(deftest test-any-on-non-empty-collection + (let [collection "things" + _ (mgcol/insert collection { :language "Clojure", :name "langohr" })] + (is (mgcol/any? "things")) + (is (mgcol/any? monger.core/*mongodb-database* "things" {:language "Clojure"})))) - (deftest basic-map-reduce-example-that-replaces-named-collection - (mgcol/remove monger.core/*mongodb-database* collection {}) - (is (mgres/ok? (mgcol/insert-batch collection batch))) - (let [output (mgcol/map-reduce collection mapper reducer "mr_outputs" {}) - results (mgcnv/from-db-object ^DBObject (.results ^MapReduceOutput output) true)] - (mgres/ok? output) - (is (= 3 (monger.core/count results))) - (is (= expected - (map #(mgcnv/from-db-object % true) (seq results)))) - (is (= expected - (map #(mgcnv/from-db-object % true) (mgcol/find "mr_outputs")))) - (.drop ^MapReduceOutput output))) +(deftest test-empty-on-empty-collection + (let [collection "things"] + (is (mgcol/empty? collection)) + (is (mgcol/empty? monger.core/*mongodb-database* collection)))) - (deftest basic-map-reduce-example-that-merged-results-into-named-collection - (mgcol/remove monger.core/*mongodb-database* collection {}) - (is (mgres/ok? (mgcol/insert-batch collection batch))) - (mgcol/map-reduce collection mapper reducer "merged_mr_outputs" MapReduceCommand$OutputType/MERGE {}) - (is (mgres/ok? (mgcol/insert collection { :state "OR" :price 17.95 :quantity 4 }))) - (let [output (mgcol/map-reduce collection mapper reducer "merged_mr_outputs" MapReduceCommand$OutputType/MERGE {})] - (mgres/ok? output) - (is (= 4 (monger.core/count (.results ^MapReduceOutput output)))) - (is (= ["CA" "IL" "NY" "OR"] - (map :_id (mgcol/find-maps "merged_mr_outputs")))) - (.drop ^MapReduceOutput output)))) +(deftest test-empty-on-non-empty-collection + (let [collection "things" + _ (mgcol/insert collection { :language "Clojure", :name "langohr" })] + (is (not (mgcol/empty? "things"))))) ;; ;; distinct ;; -(deftest distinct-values +(deftest test-distinct-values (let [collection "widgets" batch [{ :state "CA" :quantity 1 :price 199.00 } { :state "NY" :quantity 2 :price 199.00 } @@ -187,28 +132,3 @@ (mgcol/insert-batch collection batch) (is (= ["CA" "IL" "NY"] (sort (mgcol/distinct monger.core/*mongodb-database* collection :state {})))) (is (= ["CA" "NY"] (sort (mgcol/distinct collection :state { :price { $gt 100.00 } })))))) - - -;; -;; any?, empty? -;; - -(deftest any-on-empty-collection - (let [collection "things"] - (is (not (mgcol/any? collection))))) - -(deftest any-on-non-empty-collection - (let [collection "things" - _ (mgcol/insert collection { :language "Clojure", :name "langohr" })] - (is (mgcol/any? "things")) - (is (mgcol/any? monger.core/*mongodb-database* "things" {:language "Clojure"})))) - -(deftest empty-on-empty-collection - (let [collection "things"] - (is (mgcol/empty? collection)) - (is (mgcol/empty? monger.core/*mongodb-database* collection)))) - -(deftest empty-on-non-empty-collection - (let [collection "things" - _ (mgcol/insert collection { :language "Clojure", :name "langohr" })] - (is (not (mgcol/empty? "things"))))) \ No newline at end of file diff --git a/test/monger/test/indexing.clj b/test/monger/test/indexing.clj new file mode 100644 index 0000000..250352c --- /dev/null +++ b/test/monger/test/indexing.clj @@ -0,0 +1,35 @@ +(ns monger.test.map-reduce + (:import org.bson.types.ObjectId + java.util.Date) + (:require [monger core util] + [monger.collection :as mgcol] + [monger.result :as mgres] + [monger.test.helper :as helper]) + (:use clojure.test + [monger operators conversion] + monger.test.fixtures)) + +(helper/connect!) + + +;; +;; indexes +;; + +(deftest ^{:indexing true} test-creating-and-dropping-indexes + (let [collection "libraries"] + (mgcol/drop-indexes collection) + (is (= "_id_" + (:name (first (mgcol/indexes-on collection))))) + (is (nil? (second (mgcol/indexes-on collection)))) + (mgcol/create-index collection { "language" 1 }) + (is (= "language_1" + (:name (second (mgcol/indexes-on collection))))) + (mgcol/drop-index collection "language_1") + (is (nil? (second (mgcol/indexes-on collection)))) + (mgcol/ensure-index collection { "language" 1 } {:unique true}) + (is (= "language_1" + (:name (second (mgcol/indexes-on collection))))) + (mgcol/ensure-index collection { "language" 1 }) + (mgcol/ensure-index collection { "language" 1 } { :unique true }) + (mgcol/drop-indexes collection))) diff --git a/test/monger/test/map_reduce.clj b/test/monger/test/map_reduce.clj new file mode 100644 index 0000000..e7e6cc6 --- /dev/null +++ b/test/monger/test/map_reduce.clj @@ -0,0 +1,69 @@ +(ns monger.test.map-reduce + (:import [com.mongodb WriteResult WriteConcern DBCursor DBObject CommandResult$CommandFailure MapReduceOutput MapReduceCommand MapReduceCommand$OutputType] + org.bson.types.ObjectId + java.util.Date) + (:require [monger core util] + [monger.collection :as mgcol] + [monger.result :as mgres] + [monger.js :as js] + [monger.test.helper :as helper]) + (:use clojure.test + [monger operators conversion] + [monger.test.fixtures])) + +(helper/connect!) + +(use-fixtures :each purge-people purge-docs purge-things purge-libraries) + + +;; +;; Map/Reduce +;; + +(let [collection "widgets" + mapper (js/load-resource "resources/mongo/js/mapfun1.js") + reducer "function(key, values) { + var result = 0; + values.forEach(function(v) { result += v }); + + return result; + }" + batch [{ :state "CA" :quantity 1 :price 199.00 } + { :state "NY" :quantity 2 :price 199.00 } + { :state "NY" :quantity 1 :price 299.00 } + { :state "IL" :quantity 2 :price 11.50 } + { :state "CA" :quantity 2 :price 2.95 } + { :state "IL" :quantity 3 :price 5.50 }] + expected [{:_id "CA", :value 204.9} {:_id "IL", :value 39.5} {:_id "NY", :value 697.0}]] + (deftest test-basic-inline-map-reduce-example + (mgcol/remove monger.core/*mongodb-database* collection {}) + (is (mgres/ok? (mgcol/insert-batch collection batch))) + (let [output (mgcol/map-reduce collection mapper reducer nil MapReduceCommand$OutputType/INLINE {}) + results (from-db-object ^DBObject (.results ^MapReduceOutput output) true)] + (mgres/ok? output) + (is (= expected results)))) + + (deftest test-basic-map-reduce-example-that-replaces-named-collection + (mgcol/remove monger.core/*mongodb-database* collection {}) + (is (mgres/ok? (mgcol/insert-batch collection batch))) + (let [output (mgcol/map-reduce collection mapper reducer "mr_outputs" {}) + results (from-db-object ^DBObject (.results ^MapReduceOutput output) true)] + (mgres/ok? output) + (is (= 3 (monger.core/count results))) + (is (= expected + (map #(from-db-object % true) (seq results)))) + (is (= expected + (map #(from-db-object % true) (mgcol/find "mr_outputs")))) + (.drop ^MapReduceOutput output))) + + (deftest test-basic-map-reduce-example-that-merged-results-into-named-collection + (mgcol/remove monger.core/*mongodb-database* collection {}) + (is (mgres/ok? (mgcol/insert-batch collection batch))) + (mgcol/map-reduce collection mapper reducer "merged_mr_outputs" MapReduceCommand$OutputType/MERGE {}) + (is (mgres/ok? (mgcol/insert collection { :state "OR" :price 17.95 :quantity 4 }))) + (let [output (mgcol/map-reduce collection mapper reducer "merged_mr_outputs" MapReduceCommand$OutputType/MERGE {})] + (mgres/ok? output) + (is (= 4 (monger.core/count (.results ^MapReduceOutput output)))) + (is (= ["CA" "IL" "NY" "OR"] + (map :_id (mgcol/find-maps "merged_mr_outputs")))) + (.drop ^MapReduceOutput output)))) From a6d15dbbe1a5ec18a2dcb71b0ece319372e9cc17 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 10 Apr 2012 19:16:21 +0400 Subject: [PATCH 18/26] Extend monger.core/count to MapReduceOutput --- src/monger/core.clj | 8 ++++++-- test/monger/test/map_reduce.clj | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/monger/core.clj b/src/monger/core.clj index b2f0c06..5a0825a 100644 --- a/src/monger/core.clj +++ b/src/monger/core.clj @@ -14,7 +14,7 @@ monger.core (:refer-clojure :exclude [count]) (:use [monger.conversion]) - (:import [com.mongodb Mongo DB WriteConcern DBObject DBCursor CommandResult Bytes MongoOptions ServerAddress] + (:import [com.mongodb Mongo DB WriteConcern DBObject DBCursor CommandResult Bytes MongoOptions ServerAddress MapReduceOutput] [com.mongodb.gridfs GridFS] [java.util Map])) @@ -237,7 +237,11 @@ (extend-protocol Countable DBCursor (count [^DBCursor this] - (.count this))) + (.count this)) + + MapReduceOutput + (count [^MapReduceOutput this] + (.count (.results this)))) (defn ^DBObject get-last-error "Returns the the error (if there is one) from the previous operation on this connection. diff --git a/test/monger/test/map_reduce.clj b/test/monger/test/map_reduce.clj index e7e6cc6..7204fa8 100644 --- a/test/monger/test/map_reduce.clj +++ b/test/monger/test/map_reduce.clj @@ -60,10 +60,10 @@ (mgcol/remove monger.core/*mongodb-database* collection {}) (is (mgres/ok? (mgcol/insert-batch collection batch))) (mgcol/map-reduce collection mapper reducer "merged_mr_outputs" MapReduceCommand$OutputType/MERGE {}) - (is (mgres/ok? (mgcol/insert collection { :state "OR" :price 17.95 :quantity 4 }))) - (let [output (mgcol/map-reduce collection mapper reducer "merged_mr_outputs" MapReduceCommand$OutputType/MERGE {})] + (is (mgres/ok? (mgcol/insert collection { :state "OR" :price 17.95 :quantity 4 }))) + (let [^MapReduceOutput output (mgcol/map-reduce collection mapper reducer "merged_mr_outputs" MapReduceCommand$OutputType/MERGE {})] (mgres/ok? output) - (is (= 4 (monger.core/count (.results ^MapReduceOutput output)))) + (is (= 4 (monger.core/count output))) (is (= ["CA" "IL" "NY" "OR"] (map :_id (mgcol/find-maps "merged_mr_outputs")))) (.drop ^MapReduceOutput output)))) From 9ebd9d89dd1d131e315c765827a2a2397e0a3f19 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 10 Apr 2012 20:46:22 +0400 Subject: [PATCH 19/26] 1.0.0-beta3 --- ChangeLog.md | 5 +++++ README.md | 4 ++-- project.clj | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index ac55dbd..2efbb28 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,8 @@ +## Changes between 1.0.0-beta3 and 1.0.0-beta4 + +No changes yet. + + ## Changes between 1.0.0-beta2 and 1.0.0-beta3 ### Support for field negation in queries diff --git a/README.md b/README.md index 8d69329..07829f6 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ together with documentation guides and dedicated website. With Leiningen: - [com.novemberain/monger "1.0.0-beta2"] + [com.novemberain/monger "1.0.0-beta3"] With Maven: @@ -53,7 +53,7 @@ With Maven: com.novemberain monger - 1.0.0-beta2 + 1.0.0-beta3 diff --git a/project.clj b/project.clj index 44c18b9..5d71a6d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.novemberain/monger "1.0.0-SNAPSHOT" +(defproject com.novemberain/monger "1.0.0-beta3" :description "Monger is an experimental idiomatic Clojure wrapper around MongoDB Java driver" :min-lein-version "2.0.0" :license {:name "Eclipse Public License"} From efbb439dcabeed422259a795b71148a05fff0250 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 10 Apr 2012 20:47:08 +0400 Subject: [PATCH 20/26] Back to snapshot --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5d71a6d..44c18b9 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.novemberain/monger "1.0.0-beta3" +(defproject com.novemberain/monger "1.0.0-SNAPSHOT" :description "Monger is an experimental idiomatic Clojure wrapper around MongoDB Java driver" :min-lein-version "2.0.0" :license {:name "Eclipse Public License"} From 0da0a696f23a4ce1ecf6d11382633eb2049d728e Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Tue, 10 Apr 2012 23:54:23 +0400 Subject: [PATCH 21/26] Initial support for URI connections. I want to believe that the person who designed MongoDB Java driver API was not sober while at it. --- src/monger/core.clj | 32 ++++++++++++++++++++++++++++---- test/monger/test/core.clj | 4 ++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/monger/core.clj b/src/monger/core.clj index 5a0825a..7599136 100644 --- a/src/monger/core.clj +++ b/src/monger/core.clj @@ -14,7 +14,7 @@ monger.core (:refer-clojure :exclude [count]) (:use [monger.conversion]) - (:import [com.mongodb Mongo DB WriteConcern DBObject DBCursor CommandResult Bytes MongoOptions ServerAddress MapReduceOutput] + (:import [com.mongodb Mongo MongoURI DB WriteConcern DBObject DBCursor CommandResult Bytes MongoOptions ServerAddress MapReduceOutput] [com.mongodb.gridfs GridFS] [java.util Map])) @@ -31,6 +31,7 @@ (declare ^:dynamic ^GridFS *mongodb-gridfs*) + ;; ;; API ;; @@ -51,11 +52,10 @@ (Mongo.)) ([^ServerAddress server-address ^MongoOptions options] (Mongo. server-address options)) - ([{ :keys [host port] :or { host *mongodb-host*, port *mongodb-port* }}] + ([{ :keys [host port uri] :or { host *mongodb-host* port *mongodb-port* }}] (Mongo. ^String host ^Long port))) - (defn ^DB get-db-names "Gets a list of all database names present on the server" ([] @@ -105,7 +105,7 @@ (defn server-address ([^String hostname] (ServerAddress. hostname)) - ([^String hostname ^long port] + ([^String hostname ^Long port] (ServerAddress. hostname port))) @@ -172,6 +172,30 @@ and WebScale fast second." (def ^:dynamic *mongodb-write-concern* wc)) + +(defn connect-via-uri! + "Connects to MongoDB using a URI, sets up default connection and database. Commonly used for PaaS-based applications, + for example, running on Heroku. If username and password are provided, performs authentication." + [{ :keys [uri] }] + (let [uri (MongoURI. uri) + ;; yes, you are not hallucinating. A class named MongoURI has a method called connectDB. + ;; I call it "college OOP". Or maybe "don't give a shit" OOP. + db (.connectDB uri) + conn (.getMongo db) + user (.getUsername uri) + pwd (.getPassword uri)] + ;; I hope that whoever wrote the MongoDB Java driver connection/authentication parts + ;; wasn't sober while at it. MK. + ;; + ;; First we set connection, then DB, then authentcate + (set-connection! conn) + (when db + (set-db! db)) + (when (and user pwd) + (authenticate db user pwd)) + conn)) + + (defn ^CommandResult command "Runs a database command (please check MongoDB documentation for the complete list of commands). Some common commands are: diff --git a/test/monger/test/core.clj b/test/monger/test/core.clj index 29a5803..a91ead0 100644 --- a/test/monger/test/core.clj +++ b/test/monger/test/core.clj @@ -23,6 +23,10 @@ (let [connection (monger.core/connect { :host "127.0.0.1" })] (is (instance? com.mongodb.Mongo connection)))) +(deftest connect-to-mongo-via-uri-without-credentials + (let [connection (monger.core/connect "mongodb://127.0.0.1")] + (is (instance? com.mongodb.Mongo connection)))) + (deftest test-mongo-options-builder (let [max-wait-time (* 1000 60 2) From 6e7c29ba8b5361b7a87e081260a32b9e560c731b Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 00:20:07 +0400 Subject: [PATCH 22/26] Improve support for URI connections --- src/monger/core.clj | 5 +++-- test/monger/test/core.clj | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/monger/core.clj b/src/monger/core.clj index 7599136..cf86497 100644 --- a/src/monger/core.clj +++ b/src/monger/core.clj @@ -176,7 +176,7 @@ (defn connect-via-uri! "Connects to MongoDB using a URI, sets up default connection and database. Commonly used for PaaS-based applications, for example, running on Heroku. If username and password are provided, performs authentication." - [{ :keys [uri] }] + [uri] (let [uri (MongoURI. uri) ;; yes, you are not hallucinating. A class named MongoURI has a method called connectDB. ;; I call it "college OOP". Or maybe "don't give a shit" OOP. @@ -192,7 +192,8 @@ (when db (set-db! db)) (when (and user pwd) - (authenticate db user pwd)) + (when-not (authenticate (.getName db) user pwd) + (throw (IllegalArgumentException. "Could not authenticate. Either database name or credentials are invalid.")))) conn)) diff --git a/test/monger/test/core.clj b/test/monger/test/core.clj index a91ead0..6bcae50 100644 --- a/test/monger/test/core.clj +++ b/test/monger/test/core.clj @@ -24,8 +24,20 @@ (is (instance? com.mongodb.Mongo connection)))) (deftest connect-to-mongo-via-uri-without-credentials - (let [connection (monger.core/connect "mongodb://127.0.0.1")] - (is (instance? com.mongodb.Mongo connection)))) + (let [connection (monger.core/connect-via-uri! "mongodb://127.0.0.1/monger-test4")] + (is (= (-> connection .getAddress (.sameHost "127.0.0.1"))))) + ;; reconnect using regular host + (helper/connect!)) + +(deftest connect-to-mongo-via-uri-with-valid-credentials + (let [connection (monger.core/connect-via-uri! "mongodb://clojurewerkz/monger!:monger!@127.0.0.1/monger-test4")] + (is (= (-> connection .getAddress (.sameHost "127.0.0.1"))))) + ;; reconnect using regular host + (helper/connect!)) + +(deftest connect-to-mongo-via-uri-with-invalid-credentials + (is (thrown? IllegalArgumentException + (monger.core/connect-via-uri! "mongodb://clojurewerkz/monger!:ahsidaysd78jahsdi8@127.0.0.1/monger-test4")))) (deftest test-mongo-options-builder From 9a058ea45cb8b2be09b255807ea36657b72ab17e Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 00:20:11 +0400 Subject: [PATCH 23/26] Update change log --- ChangeLog.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 2efbb28..1c15baa 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,28 @@ ## Changes between 1.0.0-beta3 and 1.0.0-beta4 -No changes yet. +### Support for URI connections (and thus PaaS provides like Heroku) + +`monger.core/connect-via-uri!` is a new function that combines `monger.core/connect!`, `monger.core/set-db!` and `monger.core/authenticate` +and works with string URIs like `mongodb://userb71148a:0da0a696f23a4ce1ecf6d11382633eb2049d728e@cluster1.mongohost.com:27034/app81766662`. + +It can be used to connect with or without authentication, for example: + +``` clojure +;; connect without authentication +(monger.core/connect-via-uri! "mongodb://127.0.0.1/monger-test4") + +;; connect with authentication +(monger.core/connect-via-uri! "mongodb://clojurewerkz/monger!:monger!@127.0.0.1/monger-test4") + +;; connect using connection URI stored in an env variable, in this case, MONGOHQ_URL +(monger.core/connect-via-uri! (System/genenv "MONGOHQ_URL")) +``` + +It is also possible to pass connection options and query parameters: + +``` clojure +(monger.core/connect-via-uri! "mongodb://localhost/test?maxPoolSize=128&waitQueueMultiple=5;waitQueueTimeoutMS=150;socketTimeoutMS=5500&autoConnectRetry=true;safe=false&w=1;wtimeout=2500;fsync=true") +``` ## Changes between 1.0.0-beta2 and 1.0.0-beta3 From 19b8e48084a5c4fb94236688d5d4c98d74f62507 Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 01:33:29 +0400 Subject: [PATCH 24/26] Make sure we authenticate before calling set-db! (which instantiates a GridFS object) Fixes Heroku/MongoHQ add-on issues --- project.clj | 1 + src/monger/core.clj | 5 +++-- test/monger/test/core.clj | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 44c18b9..c1fc532 100644 --- a/project.clj +++ b/project.clj @@ -8,6 +8,7 @@ :test-selectors {:default (complement :performance) :focus :focus :indexing :indexing + :external :external :performance :performance :all (constantly true)} :codox {:exclude [monger.internal.pagination]} diff --git a/src/monger/core.clj b/src/monger/core.clj index cf86497..be62984 100644 --- a/src/monger/core.clj +++ b/src/monger/core.clj @@ -189,11 +189,12 @@ ;; ;; First we set connection, then DB, then authentcate (set-connection! conn) - (when db - (set-db! db)) (when (and user pwd) (when-not (authenticate (.getName db) user pwd) (throw (IllegalArgumentException. "Could not authenticate. Either database name or credentials are invalid.")))) + ;; only do this *after* we authenticated because set-db! will try to set up a default GridFS instance. MK. + (when db + (set-db! db)) conn)) diff --git a/test/monger/test/core.clj b/test/monger/test/core.clj index 6bcae50..d0a8b95 100644 --- a/test/monger/test/core.clj +++ b/test/monger/test/core.clj @@ -35,6 +35,14 @@ ;; reconnect using regular host (helper/connect!)) +(if-let [uri (System/getenv "MONGOHQ_URL")] + (deftest ^{:external true} connect-to-mongo-via-uri-with-valid-credentials + (let [connection (monger.core/connect-via-uri! uri)] + (is (= (-> connection .getAddress (.sameHost "127.0.0.1"))))) + ;; reconnect using regular host + (helper/connect!))) + + (deftest connect-to-mongo-via-uri-with-invalid-credentials (is (thrown? IllegalArgumentException (monger.core/connect-via-uri! "mongodb://clojurewerkz/monger!:ahsidaysd78jahsdi8@127.0.0.1/monger-test4")))) From 7ade151ecc31a8e598ba5ac2193d5f605cb338cc Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 01:42:07 +0400 Subject: [PATCH 25/26] 1.0.0-beta4 --- ChangeLog.md | 6 ++++++ README.md | 9 +++++---- project.clj | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 1c15baa..b01885c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,9 @@ +## Changes between 1.0.0-beta4 and 1.0.0-beta5 + +No changes yet. + + + ## Changes between 1.0.0-beta3 and 1.0.0-beta4 ### Support for URI connections (and thus PaaS provides like Heroku) diff --git a/README.md b/README.md index 07829f6..fd59e41 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,12 @@ wanted a client that will * Be well documented. * Be well tested. * Be maintained, do not carry technical debt from 2009 forever. + * Target Clojure 1.3.0 and later from the ground up. * Integrate with libraries like clojure.data.json and Joda Time. * Provide support for unit testing: factories/fixtures DSL, collection cleaner functions, clojure.test integration and so on. - * Integrate usage of JavaScript files and ClojureScript (as soon as the compiler gets artifact it is possible to depend on for easy embedding). + * Support URI connections to be friendly to Heroku and other PaaS providers. * Learn from other clients like the Java and Ruby ones. - * Target Clojure 1.3.0 and later from the ground up. + * Integrate usage of JavaScript files and ClojureScript (as soon as the compiler gets artifact it is possible to depend on for easy embedding). ## Documentation & Examples @@ -45,7 +46,7 @@ together with documentation guides and dedicated website. With Leiningen: - [com.novemberain/monger "1.0.0-beta3"] + [com.novemberain/monger "1.0.0-beta4"] With Maven: @@ -53,7 +54,7 @@ With Maven: com.novemberain monger - 1.0.0-beta3 + 1.0.0-beta4 diff --git a/project.clj b/project.clj index c1fc532..0eb119a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.novemberain/monger "1.0.0-SNAPSHOT" +(defproject com.novemberain/monger "1.0.0-beta4" :description "Monger is an experimental idiomatic Clojure wrapper around MongoDB Java driver" :min-lein-version "2.0.0" :license {:name "Eclipse Public License"} From ec387f15e015188dba776ae241b66b9accde169e Mon Sep 17 00:00:00 2001 From: "Michael S. Klishin" Date: Wed, 11 Apr 2012 01:43:21 +0400 Subject: [PATCH 26/26] Back to snapshot --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 0eb119a..c1fc532 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.novemberain/monger "1.0.0-beta4" +(defproject com.novemberain/monger "1.0.0-SNAPSHOT" :description "Monger is an experimental idiomatic Clojure wrapper around MongoDB Java driver" :min-lein-version "2.0.0" :license {:name "Eclipse Public License"}