diff --git a/CHANGELOG.md b/CHANGELOG.md index 258eb3a..c07721b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Only accretive/fixative changes will be made from now on. The following changes have been committed to the **master** branch since the 1.0.10 release: +* Add `next.jdbc.middleware` containing a `wrapper` for connectable objects that can offer default options, as well as four "hooks" for pre- and post-processing functions that make it easier to add logging and timing code to your `next.jdbc`-based application. * Add testing against Microsoft SQL Server (run tests with environment variables `NEXT_JDBC_TEST_MSSQL=yes` and `MSSQL_SA_PASSWORD` set to your local -- `127.0.0.1:1433` -- SQL Server `sa` user password; assumes that it can create and drop `fruit` and `fruit_time` tables in the `model` database). * Add testing against MySQL (run tests with environment variables `NEXT_JDBC_TEST_MYSQL=yes` and `MYSQL_ROOT_PASSWORD` set to your local -- `127.0.0.1:3306` -- MySQL `root` user password; assumes you have already created an empty database called `clojure_test`). * Bump several JDBC driver versions for up-to-date testing. diff --git a/src/next/jdbc/middleware.clj b/src/next/jdbc/middleware.clj new file mode 100644 index 0000000..29915c1 --- /dev/null +++ b/src/next/jdbc/middleware.clj @@ -0,0 +1,139 @@ +;; copyright (c) 2019 Sean Corfield, all rights reserved + +(ns next.jdbc.middleware + "Middleware to wrap SQL operations and result set builders. + + Can wrap a connectable such that you can: supply 'global' options for all + SQL operations on that connectable; pre-process the SQL and/or parameters + and/or the options; post-process the result set object (and options); + post-process each row as it is built; post-process the whole result set. + + The following options can be used to provide those hook functions: + * :pre-execute-fn -- pre-process the SQL & parameters and options + returns pair of (possibly updated) SQL & parameters + and (possibly updated) options + * (execute SQL) + * :post-execute-fn -- post-process the result set and options + returns pair of (possibly updated) `ResultSet` object + and (possibly updated) options + * :row!-fn -- post-process each row (and is also passed options) + returns (possibly updated) row data + * :rs!-fn -- post-process the whole result set (and is also + passed options) + returns (possibly updated) result set data + + The default for all of these is to simply return data unchanged. For + `:pre-execute-fn` and `:post-execute-fn`, that means returning a pair of + `[sql-param options]` and `[rs options]` respectively. For `:row!-fn`, + that means returning the row data unchanged (and ignoring the options). + For `:rs!-fn`, that means returning the result set data unchanged (and + ignoring the options). + + For timing middleware, you can pass per-operation timing data through the + options hash map, so you can measure the timing for the SQL execution, and + also the time taken to build the full result set (if it is built). + + For logging middleware, you get access to the SQL & parameters prior to + the execution and the full result set (if it is built). + + You can also transform the SQL & parameters prior to execution and transform + the rows and/or result set after each is built." + (:require [next.jdbc.protocols :as p] + [next.jdbc.result-set :as rs])) + +(defn post-processing-adapter + "Given a builder function (e.g., `as-lower-maps`), return a new builder + function that post-processes rows and the result set. The options may + contain post-processing functions that are called on each row and on the + the result set. The options map is provided as a second parameter to these + functions, which should include `:next.jdbc/sql-params` (the vector of SQL + and parameters, in case post-processing needs it): + + * `:post-execute-fn` -- called on the `ResultSet` object and the options + immediately after the SQL operation completes + returns a pair of a (possibly updated) `ResultSet` + object and (possibly updated) options + * `:row!-fn` -- called on each row and the options, as the row is + fully-realized and returns the (possiblly updated) + row data + * `:rs!-fn` -- called on the whole result set and the options, as + the result set is fully-realized and returns the + (possibly updated) result set data + + The results of these functions are returned as the rows/result set." + [builder-fn] + (fn [rs opts] + (let [exec-fn (get opts :post-execute-fn vector) + ;; rebind both the ResultSet object and the options + [rs opts] (exec-fn rs opts) + mrsb (builder-fn rs opts) + row!-fn (get opts :row!-fn (comp first vector)) + rs!-fn (get opts :rs!-fn (comp first vector))] + (reify + rs/RowBuilder + (->row [this] (rs/->row mrsb)) + (column-count [this] (rs/column-count mrsb)) + (with-column [this row i] (rs/with-column mrsb row i)) + (row! [this row] (row!-fn (rs/row! mrsb row) opts)) + rs/ResultSetBuilder + (->rs [this] (rs/->rs mrsb)) + (with-row [this mrs row] (rs/with-row mrsb mrs row)) + (rs! [this mrs] (rs!-fn (rs/rs! mrsb mrs) opts)))))) + +(defrecord JdbcMiddleware [db global-opts] + p/Executable + (-execute [this sql-params opts] + (let [opts (merge global-opts opts) + pre-execute-fn (get opts :pre-execute-fn vector) + ;; rebind both the SQL & parameters and the options + [sql-params opts] (pre-execute-fn sql-params opts) + builder-fn (get opts :builder-fn rs/as-maps)] + (p/-execute db sql-params + (assoc opts + :builder-fn (post-processing-adapter builder-fn) + :next.jdbc/sql-params sql-params)))) + (-execute-one [this sql-params opts] + (let [opts (merge global-opts opts) + pre-execute-fn (get opts :pre-execute-fn vector) + ;; rebind both the SQL & parameters and the options + [sql-params opts] (pre-execute-fn sql-params opts) + builder-fn (get opts :builder-fn rs/as-maps)] + (p/-execute-one db sql-params + (assoc opts + :builder-fn (post-processing-adapter builder-fn) + :next.jdbc/sql-params sql-params)))) + (-execute-all [this sql-params opts] + (let [opts (merge global-opts opts) + pre-execute-fn (get opts :pre-execute-fn vector) + ;; rebind both the SQL & parameters and the options + [sql-params opts] (pre-execute-fn sql-params opts) + builder-fn (get opts :builder-fn rs/as-maps)] + (p/-execute-all db sql-params + (assoc opts + :builder-fn (post-processing-adapter builder-fn) + :next.jdbc/sql-params sql-params))))) + +(defn wrapper + "Given a connectable, return a wrapped connectable that will run hooks. + + Given a connectable and a hash map of options, return a wrapped connectable + that will use those options as defaults for any SQL operations and will + run hooks. + + The following hooks are supported: + * :pre-execute-fn -- pre-process the SQL & parameters and options + returns pair of (possibly updated) SQL & parameters + and (possibly updated) options + * :post-execute-fn -- post-process the result set and options + returns pair of (possibly updated) `ResultSet` object + and (possibly updated) options + * :row!-fn -- post-process each row (and is also passed options) + returns (possibly updated) row data + * :rs!-fn -- post-process the whole result set (and is also + passed options) + returns (possibly updated) result set data + + Uses `next.jdbc.middleware/post-processing-adapter for the last three, + wrapped around whatever `:builder-fn` you supply for each SQL operation." + ([db] (JdbcMiddleware. db {})) + ([db opts] (JdbcMiddleware. db opts))) diff --git a/test/next/jdbc/middleware.clj b/test/next/jdbc/middleware.clj deleted file mode 100644 index aacfa4d..0000000 --- a/test/next/jdbc/middleware.clj +++ /dev/null @@ -1,115 +0,0 @@ -;; copyright (c) 2019 Sean Corfield, all rights reserved - -(ns next.jdbc.middleware - "This is just an experimental sketch of what it might look like to be - able to provide middleware that can wrap SQL execution in a way that - behavior can be extended in interesting ways, to support logging, timing. - and other cross-cutting things. - - Since it's just an experiment, there's no guarantee that this -- or - anything like it -- will actually end up in a next.jdbc release. You've - been warned! - - So far these execution points can be hooked into: - * start -- pre-process the SQL & parameters and options - * (execute SQL) - * ????? -- process the options (and something else?) - * row -- post-process each row and options - * rs -- post-process the whole result set and options - - For the rows and result set, it's 'obvious' that the functions should - take the values and return them (or updated versions). For the start - function with SQL & parameters, it also makes sense to take and return - that vector. - - For timing middleware, you'd need to pass data through the call chain - somehow -- unless you control the whole middleware and this isn't sufficient - for that yet. Hence the decision to allow processing of the options and - passing data through those -- which leads to a rather odd call chain: - start can return the vector or a map of updated options (with a payload), - and the ????? point can process the options again (e.g., to update timing - data etc). And that's all kind of horrible." - (:require [next.jdbc.protocols :as p] - [next.jdbc.result-set :as rs])) - -(defn post-processing-adapter - "Given a builder function (e.g., `as-lower-maps`), return a new builder - function that post-processes rows and the result set. The options may - contain post-processing functions that are called on each row and on the - the result set. The options map is provided as a second parameter to these - functions, which should include `:next.jdbc/sql-params` (the vector of SQL - and parameters, in case post-processing needs it): - - * `:execute-fn` -- called immediately after the SQL operation completes - ^ This is a horrible name and it needs to return the options which - is weird so I don't like this approach overall... - * `:row!-fn` -- called on each row as it is fully-realized - * `:rs!-fn` -- called on the whole result set once it is fully-realized - - The results of these functions are returned as the rows/result set." - [builder-fn] - (fn [rs opts] - (let [id2 (fn [x _] x) - id2' (fn [_ x] x) - exec-fn (get opts :execute-fn id2') - opts (exec-fn rs opts) - mrsb (builder-fn rs opts) - row!-fn (get opts :row!-fn id2) - rs!-fn (get opts :rs!-fn id2)] - (reify - rs/RowBuilder - (->row [this] (rs/->row mrsb)) - (column-count [this] (rs/column-count mrsb)) - (with-column [this row i] (rs/with-column mrsb row i)) - (row! [this row] (row!-fn (rs/row! mrsb row) opts)) - rs/ResultSetBuilder - (->rs [this] (rs/->rs mrsb)) - (with-row [this mrs row] (rs/with-row mrsb mrs row)) - (rs! [this mrs] (rs!-fn (rs/rs! mrsb mrs) opts)))))) - -(defrecord JdbcMiddleware [db global-opts] - p/Executable - (-execute [this sql-params opts] - (let [opts (merge global-opts opts) - id2 (fn [x _] x) - builder-fn (get opts :builder-fn rs/as-maps) - sql-params-fn (get opts :sql-params-fn id2) - result (sql-params-fn sql-params opts) - sql-params' (if (map? result) - (or (:next.jdbc/sql-params result) sql-params) - result)] - (p/-execute db sql-params' - (assoc (if (map? result) result opts) - :builder-fn (post-processing-adapter builder-fn) - :next.jdbc/sql-params sql-params')))) - (-execute-one [this sql-params opts] - (let [opts (merge global-opts opts) - id2 (fn [x _] x) - builder-fn (get opts :builder-fn rs/as-maps) - sql-params-fn (get opts :sql-params-fn id2) - result (sql-params-fn sql-params opts) - sql-params' (if (map? result) - (or (:next.jdbc/sql-params result) sql-params) - result)] - (p/-execute-one db sql-params' - (assoc (if (map? result) result opts) - :builder-fn (post-processing-adapter builder-fn) - :next.jdbc/sql-params sql-params')))) - (-execute-all [this sql-params opts] - (let [opts (merge global-opts opts) - id2 (fn [x _] x) - builder-fn (get opts :builder-fn rs/as-maps) - sql-params-fn (get opts :sql-params-fn id2) - result (sql-params-fn sql-params opts) - sql-params' (if (map? result) - (or (:next.jdbc/sql-params result) sql-params) - result)] - (p/-execute-all db sql-params' - (assoc (if (map? result) result opts) - :builder-fn (post-processing-adapter builder-fn) - :next.jdbc/sql-params sql-params'))))) - -(defn wrapper - "" - ([db] (JdbcMiddleware. db {})) - ([db opts] (JdbcMiddleware. db opts))) diff --git a/test/next/jdbc/middleware_test.clj b/test/next/jdbc/middleware_test.clj index 754e1c9..174fc49 100644 --- a/test/next/jdbc/middleware_test.clj +++ b/test/next/jdbc/middleware_test.clj @@ -3,12 +3,9 @@ (ns next.jdbc.middleware-test (:require [clojure.test :refer [deftest is testing use-fixtures]] [next.jdbc :as jdbc] - [next.jdbc.connection :as c] [next.jdbc.middleware :as mw] [next.jdbc.test-fixtures :refer [with-test-db db ds - default-options - derby? postgres?]] - [next.jdbc.prepare :as prep] + default-options]] [next.jdbc.result-set :as rs] [next.jdbc.specs :as specs]) (:import (java.sql ResultSet ResultSetMetaData))) @@ -21,16 +18,16 @@ (deftest logging-test (let [logging (atom []) - logger (fn [data _] (swap! logging conj data) data) - + logger (fn [data _] (swap! logging conj data) data) + log-sql (fn [sql-p opts] (logger sql-p opts) [sql-p opts]) sql-p ["select * from fruit where id in (?,?) order by id desc" 1 4]] (jdbc/execute! (mw/wrapper (ds)) sql-p (assoc (default-options) - :builder-fn rs/as-lower-maps - :sql-params-fn logger - :row!-fn logger - :rs!-fn logger)) + :builder-fn rs/as-lower-maps + :pre-execute-fn log-sql + :row!-fn logger + :rs!-fn logger)) ;; should log four things (is (= 4 (-> @logging count))) ;; :next.jdbc/sql-params value @@ -45,9 +42,9 @@ ;; now repeat without the row logging (reset! logging []) (jdbc/execute! (mw/wrapper (ds) - {:builder-fn rs/as-lower-maps - :sql-params-fn logger - :rs!-fn logger}) + {:builder-fn rs/as-lower-maps + :pre-execute-fn log-sql + :rs!-fn logger}) sql-p (default-options)) ;; should log two things @@ -59,21 +56,22 @@ (is (= [4 1] (-> @logging (nth 1) (->> (map :fruit/id))))))) (deftest timing-test - (let [timing (atom {:calls 0 :total 0.0}) - start-fn (fn [sql-p opts] - (swap! (:timing opts) update :calls inc) - (assoc opts :start (System/nanoTime))) - exec-fn (fn [_ opts] + (let [start-fn (fn [sql-p opts] + (swap! (::timing opts) update ::calls inc) + [sql-p (assoc opts ::start (System/nanoTime))]) + end-fn (fn [rs opts] (let [end (System/nanoTime)] - (swap! (:timing opts) update :total + (- end (:start opts))) - opts)) + (swap! (::timing opts) update ::total + (- end (::start opts))) + [rs opts])) + timing (atom {::calls 0 ::total 0.0}) sql-p ["select * from fruit where id in (?,?) order by id desc" 1 4]] - (jdbc/execute! (mw/wrapper (ds) {:timing timing - :sql-params-fn start-fn - :execute-fn exec-fn}) + (jdbc/execute! (mw/wrapper (ds) {::timing timing + :pre-execute-fn start-fn + :post-execute-fn end-fn}) sql-p) - (jdbc/execute! (mw/wrapper (ds) {:timing timing - :sql-params-fn start-fn - :execute-fn exec-fn}) + (jdbc/execute! (mw/wrapper (ds) {::timing timing + :pre-execute-fn start-fn + :post-execute-fn end-fn}) sql-p) - (println (db) (:calls @timing) "calls took" (long (:total @timing)) "nanoseconds"))) + (printf "%20s - %d calls took %,10d nanoseconds\n" + (:dbtype (db)) (::calls @timing) (long (::total @timing)))))