From 56eebda3fd1a0e5ecbdd8312c3e038779fc7a721 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Sat, 10 May 2014 17:34:22 -0400 Subject: [PATCH] GridFS tests now pass, plus API improvements that make sense for 2.0 --- src/clojure/monger/core.clj | 13 +- src/clojure/monger/gridfs.clj | 52 +++--- test/monger/test/gridfs_test.clj | 284 +++++++++++++++---------------- 3 files changed, 184 insertions(+), 165 deletions(-) diff --git a/src/clojure/monger/core.clj b/src/clojure/monger/core.clj index d467768..ff71224 100644 --- a/src/clojure/monger/core.clj +++ b/src/clojure/monger/core.clj @@ -38,7 +38,7 @@ ;; API ;; -(defn ^com.mongodb.MongoClient connect +(defn ^MongoClient connect "Connects to MongoDB. When used without arguments, connects to Arguments: @@ -79,7 +79,7 @@ (set (.getDatabaseNames conn))) -(defn ^com.mongodb.DB get-db +(defn ^DB get-db "Get database reference by name. EXAMPLES @@ -93,6 +93,15 @@ [^MongoClient conn ^String db] (.dropDatabase conn db)) +(defn ^GridFS get-gridfs + "Get GridFS for the given database. + + EXAMPLES + + (monger.core/get-gridfs connection \"myapp_production\")" + [^MongoClient conn ^String name] + (GridFS. (.getDB conn name))) + (defn server-address ([^String hostname] (ServerAddress. hostname)) diff --git a/src/clojure/monger/gridfs.clj b/src/clojure/monger/gridfs.clj index 8aa664c..c0d15f2 100644 --- a/src/clojure/monger/gridfs.clj +++ b/src/clojure/monger/gridfs.clj @@ -20,7 +20,7 @@ (:import [com.mongodb DB DBObject] org.bson.types.ObjectId [com.mongodb.gridfs GridFS GridFSInputFile] - [java.io InputStream File])) + [java.io InputStream ByteArrayInputStream File])) ;; ;; Implementation @@ -50,42 +50,49 @@ (remove fs {})) (defn all-files - [^GridFS fs query] - (.getFileList fs query)) + ([^GridFS fs] + (.getFileList fs (to-db-object {}))) + ([^GridFS fs query] + (.getFileList fs query))) (def ^{:private true} converter (fpartial from-db-object true)) (defn files-as-maps - [^GridFS fs query] - (map converter (all-files fs (to-db-object query)))) + ([^GridFS fs] + (files-as-maps fs {})) + ([^GridFS fs query] + (map converter (all-files fs (to-db-object query))))) ;; ;; Plumbing (low-level API) ;; -(defprotocol GridFSInputFileFactory - (^com.mongodb.gridfs.GridFSInputFile make-input-file [input] "Makes GridFSInputFile out of the given input")) +(defprotocol InputStreamFactory + (^InputStream to-input-stream [input] "Makes InputStream out of the given input")) (extend byte-array-type - GridFSInputFileFactory - {:make-input-file (fn [^bytes input] - (.createFile ^GridFS monger.core/*mongodb-gridfs* input))}) + InputStreamFactory + {:to-input-stream (fn [^bytes input] + (ByteArrayInputStream. input))}) -(extend-protocol GridFSInputFileFactory +(extend-protocol InputStreamFactory String - (make-input-file [^String input] - (.createFile ^GridFS monger.core/*mongodb-gridfs* ^InputStream (io/make-input-stream input {:encoding "UTF-8"}))) + (to-input-stream [^String input] + (io/make-input-stream input {:encoding "UTF-8"})) File - (make-input-file [^File input] - (.createFile ^GridFS monger.core/*mongodb-gridfs* ^InputStream (io/make-input-stream input {:encoding "UTF-8"}))) + (to-input-stream [^File input] + (io/make-input-stream input {:encoding "UTF-8"})) InputStream - (make-input-file [^InputStream input] - (.createFile ^GridFS monger.core/*mongodb-gridfs* ^InputStream input))) + (to-input-stream [^InputStream input] + input)) +(defn ^GridFSInputFile make-input-file + [^GridFS fs input] + (.createFile fs (to-input-stream input))) (defmacro store [^GridFSInputFile input & body] @@ -93,9 +100,8 @@ (.save f# GridFS/DEFAULT_CHUNKSIZE) (from-db-object f# true))) - ;; -;; "New" DSL, a higher-level API +;; Higher-level API ;; (defn save @@ -132,6 +138,14 @@ [^GridFS fs query] (.find fs (to-db-object query))) +(defn find-by-filename + [^GridFS fs ^String filename] + (.find fs (to-db-object {"filename" filename}))) + +(defn find-by-md5 + [^GridFS fs ^String md5] + (.find fs (to-db-object {"md5" md5}))) + (defn find-one [^GridFS fs query] (.findOne fs (to-db-object query))) diff --git a/test/monger/test/gridfs_test.clj b/test/monger/test/gridfs_test.clj index 0132fd1..9326e49 100644 --- a/test/monger/test/gridfs_test.clj +++ b/test/monger/test/gridfs_test.clj @@ -1,167 +1,163 @@ (ns monger.test.gridfs-test (:refer-clojure :exclude [count remove find]) (:require [monger.gridfs :as gridfs] - [monger.test.helper :as helper] [clojure.java.io :as io] [clojure.test :refer :all] - [monger.core :refer [count]] - [monger.test.fixtures :refer :all] + [monger.core :as mg :refer [count]] [monger.operators :refer :all] [monger.conversion :refer :all] [monger.gridfs :refer [store make-input-file store-file filename content-type metadata]]) (:import [java.io InputStream File FileInputStream] [com.mongodb.gridfs GridFS GridFSInputFile GridFSDBFile])) -(defn purge-gridfs* - [] - (gridfs/remove-all)) +(let [conn (mg/connect) + db (mg/get-db conn "monger-test") + fs (mg/get-gridfs conn "monger-test")] + (defn purge-gridfs* + [] + (gridfs/remove-all fs)) -(defn purge-gridfs - [f] - (gridfs/remove-all) - (f) - (gridfs/remove-all)) + (defn purge-gridfs + [f] + (gridfs/remove-all fs) + (f) + (gridfs/remove-all fs)) -(use-fixtures :each purge-gridfs) + (use-fixtures :each purge-gridfs) -(helper/connect!) + (deftest ^{:gridfs true} test-storing-files-to-gridfs-using-relative-fs-paths + (let [input "./test/resources/mongo/js/mapfun1.js"] + (is (= 0 (count (gridfs/all-files fs)))) + (store (make-input-file fs input) + (.setFilename "monger.test.gridfs.file1") + (.setContentType "application/octet-stream")) + (is (= 1 (count (gridfs/all-files fs)))))) + + + (deftest ^{:gridfs true} test-storing-files-to-gridfs-using-file-instances + (let [input (io/as-file "./test/resources/mongo/js/mapfun1.js")] + (is (= 0 (count (gridfs/all-files fs)))) + (store-file (make-input-file fs input) + (filename "monger.test.gridfs.file2") + (content-type "application/octet-stream")) + (is (= 1 (count (gridfs/all-files fs)))))) + + (deftest ^{:gridfs true} test-storing-bytes-to-gridfs + (let [input (.getBytes "A string") + md {:format "raw" :source "AwesomeCamera D95"} + fname "monger.test.gridfs.file3" + ct "application/octet-stream"] + (is (= 0 (count (gridfs/all-files fs)))) + (store-file (make-input-file fs input) + (filename fname) + (metadata md) + (content-type "application/octet-stream")) + (let [f (first (gridfs/files-as-maps fs))] + (is (= ct (:contentType f))) + (is (= fname (:filename f))) + (is (= md (:metadata f)))) + (is (= 1 (count (gridfs/all-files fs)))))) + + (deftest ^{:gridfs true} test-storing-files-to-gridfs-using-absolute-fs-paths + (let [tmp-file (File/createTempFile "monger.test.gridfs" "test-storing-files-to-gridfs-using-absolute-fs-paths") + _ (spit tmp-file "Some content") + input (.getAbsolutePath tmp-file)] + (is (= 0 (count (gridfs/all-files fs)))) + (store-file (make-input-file fs input) + (filename "monger.test.gridfs.file4") + (content-type "application/octet-stream")) + (is (= 1 (count (gridfs/all-files fs)))))) + + (deftest ^{:gridfs true} test-storing-files-to-gridfs-using-input-stream + (let [tmp-file (File/createTempFile "monger.test.gridfs" "test-storing-files-to-gridfs-using-input-stream") + _ (spit tmp-file "Some other content")] + (is (= 0 (count (gridfs/all-files fs)))) + (store-file fs + (make-input-file (FileInputStream. tmp-file)) + (filename "monger.test.gridfs.file4b") + (content-type "application/octet-stream")) + (is (= 1 (count (gridfs/all-files fs)))))) -(deftest ^{:gridfs true} test-storing-files-to-gridfs-using-relative-fs-paths - (let [input "./test/resources/mongo/js/mapfun1.js"] - (is (= 0 (count (gridfs/all-files)))) - (store (make-input-file input) - (.setFilename "monger.test.gridfs.file1") - (.setContentType "application/octet-stream")) - (is (= 1 (count (gridfs/all-files)))))) + (deftest ^{:gridfs true} test-finding-individual-files-on-gridfs + (testing "gridfs/find-one" + (purge-gridfs*) + (let [input "./test/resources/mongo/js/mapfun1.js" + ct "binary/octet-stream" + fname "monger.test.gridfs.file5" + md5 "14a09deabb50925a3381315149017bbd" + stored (store-file (make-input-file fs input) + (filename fname) + (content-type ct))] + (is (= 1 (count (gridfs/all-files fs)))) + (is (:_id stored)) + (is (:uploadDate stored)) + (is (= 62 (:length stored))) + (is (= md5 (:md5 stored))) + (is (= fname (:filename stored))) + (is (= ct (:contentType stored))) + (are [a b] (is (= a (:md5 (from-db-object (gridfs/find-one fs b) true)))) + md5 {:_id (:_id stored)} + md5 (to-db-object {:md5 md5})))) + (testing "gridfs/find-one-as-map" + (purge-gridfs*) + (let [input "./test/resources/mongo/js/mapfun1.js" + ct "binary/octet-stream" + fname "monger.test.gridfs.file6" + md5 "14a09deabb50925a3381315149017bbd" + stored (store-file (make-input-file fs input) + (filename fname) + (metadata (to-db-object {:meta "data"})) + (content-type ct))] + (is (= 1 (count (gridfs/all-files fs)))) + (is (:_id stored)) + (is (:uploadDate stored)) + (is (= 62 (:length stored))) + (is (= md5 (:md5 stored))) + (is (= fname (:filename stored))) + (is (= ct (:contentType stored))) + (let [m (gridfs/find-one-as-map fs {:filename fname})] + (is (= {:meta "data"} (:metadata m)))) + (are [a query] (is (= a (:md5 (gridfs/find-one-as-map fs query)))) + md5 {:_id (:_id stored)} + md5 {:md5 md5})))) - -(deftest ^{:gridfs true} test-storing-files-to-gridfs-using-file-instances - (let [input (io/as-file "./test/resources/mongo/js/mapfun1.js")] - (is (= 0 (count (gridfs/all-files)))) - (store-file (make-input-file input) - (filename "monger.test.gridfs.file2") - (content-type "application/octet-stream")) - (is (= 1 (count (gridfs/all-files)))))) - -(deftest ^{:gridfs true} test-storing-bytes-to-gridfs - (let [input (.getBytes "A string") - md {:format "raw" :source "AwesomeCamera D95"} - fname "monger.test.gridfs.file3" - ct "application/octet-stream"] - (is (= 0 (count (gridfs/all-files)))) - (store-file (make-input-file input) - (filename fname) - (metadata md) - (content-type "application/octet-stream")) - (let [f (first (gridfs/files-as-maps))] - (is (= ct (:contentType f))) - (is (= fname (:filename f))) - (is (= md (:metadata f)))) - (is (= 1 (count (gridfs/all-files)))))) - -(deftest ^{:gridfs true} test-storing-files-to-gridfs-using-absolute-fs-paths - (let [tmp-file (File/createTempFile "monger.test.gridfs" "test-storing-files-to-gridfs-using-absolute-fs-paths") - _ (spit tmp-file "Some content") - input (.getAbsolutePath tmp-file)] - (is (= 0 (count (gridfs/all-files)))) - (store-file (make-input-file input) - (filename "monger.test.gridfs.file4") - (content-type "application/octet-stream")) - (is (= 1 (count (gridfs/all-files)))))) - -(deftest ^{:gridfs true} test-storing-files-to-gridfs-using-input-stream - (let [tmp-file (File/createTempFile "monger.test.gridfs" "test-storing-files-to-gridfs-using-input-stream") - _ (spit tmp-file "Some other content")] - (is (= 0 (count (gridfs/all-files)))) - (store-file (make-input-file (FileInputStream. tmp-file)) - (filename "monger.test.gridfs.file4b") - (content-type "application/octet-stream")) - (is (= 1 (count (gridfs/all-files)))))) - - - -(deftest ^{:gridfs true} test-finding-individual-files-on-gridfs - (testing "gridfs/find-one" - (purge-gridfs*) - (let [input "./test/resources/mongo/js/mapfun1.js" - ct "binary/octet-stream" - fname "monger.test.gridfs.file5" - md5 "14a09deabb50925a3381315149017bbd" - stored (store-file (make-input-file input) - (filename fname) - (content-type ct))] - (is (= 1 (count (gridfs/all-files)))) - (is (:_id stored)) - (is (:uploadDate stored)) - (is (= 62 (:length stored))) - (is (= md5 (:md5 stored))) - (is (= fname (:filename stored))) - (is (= ct (:contentType stored))) - (are [a b] (is (= a (:md5 (from-db-object (gridfs/find-one b) true)))) - md5 (:_id stored) - md5 fname - md5 (to-db-object {:md5 md5})))) - (testing "gridfs/find-one-as-map" - (purge-gridfs*) + (deftest ^{:gridfs true} test-finding-multiple-files-on-gridfs (let [input "./test/resources/mongo/js/mapfun1.js" ct "binary/octet-stream" - fname "monger.test.gridfs.file6" md5 "14a09deabb50925a3381315149017bbd" - stored (store-file (make-input-file input) - (filename fname) - (metadata (to-db-object {:meta "data"})) - (content-type ct))] - (is (= 1 (count (gridfs/all-files)))) - (is (:_id stored)) - (is (:uploadDate stored)) - (is (= 62 (:length stored))) - (is (= md5 (:md5 stored))) - (is (= fname (:filename stored))) - (is (= ct (:contentType stored))) - (let [m (gridfs/find-one-as-map {:filename fname})] - (is (= {:meta "data"} (:metadata m)))) - (are [a query] (is (= a (:md5 (gridfs/find-one-as-map query)))) - md5 (:_id stored) - md5 fname - md5 {:md5 md5})))) - -(deftest ^{:gridfs true} test-finding-multiple-files-on-gridfs - (let [input "./test/resources/mongo/js/mapfun1.js" - ct "binary/octet-stream" - md5 "14a09deabb50925a3381315149017bbd" - stored1 (store-file (make-input-file input) - (filename "monger.test.gridfs.file6") - (content-type ct)) - stored2 (store-file (make-input-file input) - (filename "monger.test.gridfs.file7") - (content-type ct)) - list1 (gridfs/find "monger.test.gridfs.file6") - list2 (gridfs/find "monger.test.gridfs.file7") - list3 (gridfs/find "888000___.monger.test.gridfs.file") - list4 (gridfs/find { :md5 md5 })] - (is (= 2 (count (gridfs/all-files)))) - (are [a b] (is (= (map #(.get ^GridFSDBFile % "_id") a) - (map :_id b))) - list1 [stored1] - list2 [stored2] - list3 [] - list4 [stored1 stored2]))) + stored1 (store-file (make-input-file fs input) + (filename "monger.test.gridfs.file6") + (content-type ct)) + stored2 (store-file (make-input-file fs input) + (filename "monger.test.gridfs.file7") + (content-type ct)) + list1 (gridfs/find-by-filename fs "monger.test.gridfs.file6") + list2 (gridfs/find-by-filename fs "monger.test.gridfs.file7") + list3 (gridfs/find-by-filename fs "888000___.monger.test.gridfs.file") + list4 (gridfs/find-by-md5 fs md5)] + (is (= 2 (count (gridfs/all-files fs)))) + (are [a b] (is (= (map #(.get ^GridFSDBFile % "_id") a) + (map :_id b))) + list1 [stored1] + list2 [stored2] + list3 [] + list4 [stored1 stored2]))) -(deftest ^{:gridfs true} test-removing-multiple-files-from-gridfs - (let [input "./test/resources/mongo/js/mapfun1.js" - ct "binary/octet-stream" - md5 "14a09deabb50925a3381315149017bbd" - stored1 (store-file (make-input-file input) - (filename "monger.test.gridfs.file8") - (content-type ct)) - stored2 (store-file (make-input-file input) - (filename "monger.test.gridfs.file9") - (content-type ct))] - (is (= 2 (count (gridfs/all-files)))) - (gridfs/remove { :filename "monger.test.gridfs.file8" }) - (is (= 1 (count (gridfs/all-files)))) - (gridfs/remove { :md5 md5 }) - (is (= 0 (count (gridfs/all-files)))))) + (deftest ^{:gridfs true} test-removing-multiple-files-from-gridfs + (let [input "./test/resources/mongo/js/mapfun1.js" + ct "binary/octet-stream" + md5 "14a09deabb50925a3381315149017bbd" + stored1 (store-file (make-input-file fs input) + (filename "monger.test.gridfs.file8") + (content-type ct)) + stored2 (store-file (make-input-file fs input) + (filename "monger.test.gridfs.file9") + (content-type ct))] + (is (= 2 (count (gridfs/all-files fs)))) + (gridfs/remove fs { :filename "monger.test.gridfs.file8" }) + (is (= 1 (count (gridfs/all-files fs)))) + (gridfs/remove fs { :md5 md5 }) + (is (= 0 (count (gridfs/all-files fs)))))))