First cut of middleware in release-worthy state

CHANGELOG.md

@@ -6,6 +6,7 @@ Only accretive/fixative changes will be made from now on.
 
 The following changes have been committed to the **master** branch since the 1.0.10 release:
 
+* Add `next.jdbc.middleware` containing a `wrapper` for connectable objects that can offer default options, as well as four "hooks" for pre- and post-processing functions that make it easier to add logging and timing code to your `next.jdbc`-based application.
 * Add testing against Microsoft SQL Server (run tests with environment variables `NEXT_JDBC_TEST_MSSQL=yes` and `MSSQL_SA_PASSWORD` set to your local -- `127.0.0.1:1433` -- SQL Server `sa` user password; assumes that it can create and drop `fruit` and `fruit_time` tables in the `model` database).
 * Add testing against MySQL (run tests with environment variables `NEXT_JDBC_TEST_MYSQL=yes` and `MYSQL_ROOT_PASSWORD` set to your local -- `127.0.0.1:3306` -- MySQL `root` user password; assumes you have already created an empty database called `clojure_test`).
 * Bump several JDBC driver versions for up-to-date testing.

src/next/jdbc/middleware.clj

@@ -0,0 +1,139 @@
+;; copyright (c) 2019 Sean Corfield, all rights reserved
+
+(ns next.jdbc.middleware
+  "Middleware to wrap SQL operations and result set builders.
+
+  Can wrap a connectable such that you can: supply 'global' options for all
+  SQL operations on that connectable; pre-process the SQL and/or parameters
+  and/or the options; post-process the result set object (and options);
+  post-process each row as it is built; post-process the whole result set.
+
+  The following options can be used to provide those hook functions:
+  * :pre-execute-fn  -- pre-process the SQL & parameters and options
+                        returns pair of (possibly updated) SQL & parameters
+                        and (possibly updated) options
+  * (execute SQL)
+  * :post-execute-fn -- post-process the result set and options
+                        returns pair of (possibly updated) `ResultSet` object
+                        and (possibly updated) options
+  * :row!-fn         -- post-process each row (and is also passed options)
+                        returns (possibly updated) row data
+  * :rs!-fn          -- post-process the whole result set (and is also
+                        passed options)
+                        returns (possibly updated) result set data
+
+  The default for all of these is to simply return data unchanged. For
+  `:pre-execute-fn` and `:post-execute-fn`, that means returning a pair of
+  `[sql-param options]` and `[rs options]` respectively. For `:row!-fn`,
+  that means returning the row data unchanged (and ignoring the options).
+  For `:rs!-fn`, that means returning the result set data unchanged (and
+  ignoring the options).
+
+  For timing middleware, you can pass per-operation timing data through the
+  options hash map, so you can measure the timing for the SQL execution, and
+  also the time taken to build the full result set (if it is built).
+
+  For logging middleware, you get access to the SQL & parameters prior to
+  the execution and the full result set (if it is built).
+
+  You can also transform the SQL & parameters prior to execution and transform
+  the rows and/or result set after each is built."
+  (:require [next.jdbc.protocols :as p]
+            [next.jdbc.result-set :as rs]))
+
+(defn post-processing-adapter
+  "Given a builder function (e.g., `as-lower-maps`), return a new builder
+  function that post-processes rows and the result set. The options may
+  contain post-processing functions that are called on each row and on the
+  the result set. The options map is provided as a second parameter to these
+  functions, which should include `:next.jdbc/sql-params` (the vector of SQL
+  and parameters, in case post-processing needs it):
+
+  * `:post-execute-fn` -- called on the `ResultSet` object and the options
+                          immediately after the SQL operation completes
+                          returns a pair of a (possibly updated) `ResultSet`
+                          object and (possibly updated) options
+  * `:row!-fn`         -- called on each row and the options, as the row is
+                          fully-realized and returns the (possiblly updated)
+                          row data
+  * `:rs!-fn`          -- called on the whole result set and the options, as
+                          the result set is fully-realized and returns the
+                          (possibly updated) result set data
+
+  The results of these functions are returned as the rows/result set."
+  [builder-fn]
+  (fn [rs opts]
+    (let [exec-fn   (get opts :post-execute-fn vector)
+          ;; rebind both the ResultSet object and the options
+          [rs opts] (exec-fn rs opts)
+          mrsb      (builder-fn rs opts)
+          row!-fn   (get opts :row!-fn (comp first vector))
+          rs!-fn    (get opts :rs!-fn  (comp first vector))]
+      (reify
+        rs/RowBuilder
+        (->row [this] (rs/->row mrsb))
+        (column-count [this] (rs/column-count mrsb))
+        (with-column [this row i] (rs/with-column mrsb row i))
+        (row! [this row] (row!-fn (rs/row! mrsb row) opts))
+        rs/ResultSetBuilder
+        (->rs [this] (rs/->rs mrsb))
+        (with-row [this mrs row] (rs/with-row mrsb mrs row))
+        (rs! [this mrs] (rs!-fn (rs/rs! mrsb mrs) opts))))))
+
+(defrecord JdbcMiddleware [db global-opts]
+  p/Executable
+  (-execute [this sql-params opts]
+    (let [opts              (merge global-opts opts)
+          pre-execute-fn    (get opts :pre-execute-fn vector)
+          ;; rebind both the SQL & parameters and the options
+          [sql-params opts] (pre-execute-fn sql-params opts)
+          builder-fn        (get opts :builder-fn rs/as-maps)]
+      (p/-execute db sql-params
+                  (assoc opts
+                         :builder-fn (post-processing-adapter builder-fn)
+                         :next.jdbc/sql-params sql-params))))
+  (-execute-one [this sql-params opts]
+    (let [opts              (merge global-opts opts)
+          pre-execute-fn    (get opts :pre-execute-fn vector)
+          ;; rebind both the SQL & parameters and the options
+          [sql-params opts] (pre-execute-fn sql-params opts)
+          builder-fn        (get opts :builder-fn rs/as-maps)]
+      (p/-execute-one db sql-params
+                      (assoc opts
+                             :builder-fn (post-processing-adapter builder-fn)
+                             :next.jdbc/sql-params sql-params))))
+  (-execute-all [this sql-params opts]
+    (let [opts              (merge global-opts opts)
+          pre-execute-fn    (get opts :pre-execute-fn vector)
+          ;; rebind both the SQL & parameters and the options
+          [sql-params opts] (pre-execute-fn sql-params opts)
+          builder-fn        (get opts :builder-fn rs/as-maps)]
+      (p/-execute-all db sql-params
+                      (assoc opts
+                             :builder-fn (post-processing-adapter builder-fn)
+                             :next.jdbc/sql-params sql-params)))))
+
+(defn wrapper
+  "Given a connectable, return a wrapped connectable that will run hooks.
+
+  Given a connectable and a hash map of options, return a wrapped connectable
+  that will use those options as defaults for any SQL operations and will
+  run hooks.
+
+  The following hooks are supported:
+  * :pre-execute-fn  -- pre-process the SQL & parameters and options
+                        returns pair of (possibly updated) SQL & parameters
+                        and (possibly updated) options
+  * :post-execute-fn -- post-process the result set and options
+                        returns pair of (possibly updated) `ResultSet` object
+                        and (possibly updated) options
+  * :row!-fn         -- post-process each row (and is also passed options)
+                        returns (possibly updated) row data
+  * :rs!-fn          -- post-process the whole result set (and is also
+                        passed options)
+                        returns (possibly updated) result set data
+
+  Uses `next.jdbc.middleware/post-processing-adapter for the last three,
+  wrapped around whatever `:builder-fn` you supply for each SQL operation."
+  ([db]      (JdbcMiddleware. db {}))
+  ([db opts] (JdbcMiddleware. db opts)))

test/next/jdbc/middleware.clj

@@ -1,115 +0,0 @@
-;; copyright (c) 2019 Sean Corfield, all rights reserved
-
-(ns next.jdbc.middleware
-  "This is just an experimental sketch of what it might look like to be
-  able to provide middleware that can wrap SQL execution in a way that
-  behavior can be extended in interesting ways, to support logging, timing.
-  and other cross-cutting things.
-
-  Since it's just an experiment, there's no guarantee that this -- or
-  anything like it -- will actually end up in a next.jdbc release. You've
-  been warned!
-
-  So far these execution points can be hooked into:
-  * start -- pre-process the SQL & parameters and options
-  * (execute SQL)
-  * ????? -- process the options (and something else?)
-  * row   -- post-process each row and options
-  * rs    -- post-process the whole result set and options
-
-  For the rows and result set, it's 'obvious' that the functions should
-  take the values and return them (or updated versions). For the start
-  function with SQL & parameters, it also makes sense to take and return
-  that vector.
-
-  For timing middleware, you'd need to pass data through the call chain
-  somehow -- unless you control the whole middleware and this isn't sufficient
-  for that yet. Hence the decision to allow processing of the options and
-  passing data through those -- which leads to a rather odd call chain:
-  start can return the vector or a map of updated options (with a payload),
-  and the ????? point can process the options again (e.g., to update timing
-  data etc). And that's all kind of horrible."
-  (:require [next.jdbc.protocols :as p]
-            [next.jdbc.result-set :as rs]))
-
-(defn post-processing-adapter
-  "Given a builder function (e.g., `as-lower-maps`), return a new builder
-  function that post-processes rows and the result set. The options may
-  contain post-processing functions that are called on each row and on the
-  the result set. The options map is provided as a second parameter to these
-  functions, which should include `:next.jdbc/sql-params` (the vector of SQL
-  and parameters, in case post-processing needs it):
-
-  * `:execute-fn` -- called immediately after the SQL operation completes
-      ^ This is a horrible name and it needs to return the options which
-        is weird so I don't like this approach overall...
-  * `:row!-fn` -- called on each row as it is fully-realized
-  * `:rs!-fn` -- called on the whole result set once it is fully-realized
-
-  The results of these functions are returned as the rows/result set."
-  [builder-fn]
-  (fn [rs opts]
-    (let [id2     (fn [x _] x)
-          id2'    (fn [_ x] x)
-          exec-fn (get opts :execute-fn id2')
-          opts    (exec-fn rs opts)
-          mrsb    (builder-fn rs opts)
-          row!-fn (get opts :row!-fn id2)
-          rs!-fn  (get opts :rs!-fn id2)]
-      (reify
-        rs/RowBuilder
-        (->row [this] (rs/->row mrsb))
-        (column-count [this] (rs/column-count mrsb))
-        (with-column [this row i] (rs/with-column mrsb row i))
-        (row! [this row] (row!-fn (rs/row! mrsb row) opts))
-        rs/ResultSetBuilder
-        (->rs [this] (rs/->rs mrsb))
-        (with-row [this mrs row] (rs/with-row mrsb mrs row))
-        (rs! [this mrs] (rs!-fn (rs/rs! mrsb mrs) opts))))))
-
-(defrecord JdbcMiddleware [db global-opts]
-  p/Executable
-  (-execute [this sql-params opts]
-    (let [opts          (merge global-opts opts)
-          id2           (fn [x _] x)
-          builder-fn    (get opts :builder-fn rs/as-maps)
-          sql-params-fn (get opts :sql-params-fn id2)
-          result        (sql-params-fn sql-params opts)
-          sql-params'   (if (map? result)
-                          (or (:next.jdbc/sql-params result) sql-params)
-                          result)]
-      (p/-execute db sql-params'
-                  (assoc (if (map? result) result opts)
-                         :builder-fn (post-processing-adapter builder-fn)
-                         :next.jdbc/sql-params sql-params'))))
-  (-execute-one [this sql-params opts]
-    (let [opts          (merge global-opts opts)
-          id2           (fn [x _] x)
-          builder-fn    (get opts :builder-fn rs/as-maps)
-          sql-params-fn (get opts :sql-params-fn id2)
-          result        (sql-params-fn sql-params opts)
-          sql-params'   (if (map? result)
-                          (or (:next.jdbc/sql-params result) sql-params)
-                          result)]
-      (p/-execute-one db sql-params'
-                      (assoc (if (map? result) result opts)
-                             :builder-fn (post-processing-adapter builder-fn)
-                             :next.jdbc/sql-params sql-params'))))
-  (-execute-all [this sql-params opts]
-    (let [opts          (merge global-opts opts)
-          id2           (fn [x _] x)
-          builder-fn    (get opts :builder-fn rs/as-maps)
-          sql-params-fn (get opts :sql-params-fn id2)
-          result        (sql-params-fn sql-params opts)
-          sql-params'   (if (map? result)
-                          (or (:next.jdbc/sql-params result) sql-params)
-                          result)]
-      (p/-execute-all db sql-params'
-                      (assoc (if (map? result) result opts)
-                             :builder-fn (post-processing-adapter builder-fn)
-                             :next.jdbc/sql-params sql-params')))))
-
-(defn wrapper
-  ""
-  ([db]      (JdbcMiddleware. db {}))
-  ([db opts] (JdbcMiddleware. db opts)))

test/next/jdbc/middleware_test.clj

@@ -3,12 +3,9 @@
 (ns next.jdbc.middleware-test
   (:require [clojure.test :refer [deftest is testing use-fixtures]]
             [next.jdbc :as jdbc]
-            [next.jdbc.connection :as c]
             [next.jdbc.middleware :as mw]
             [next.jdbc.test-fixtures :refer [with-test-db db ds
-                                              default-options
+                                              default-options]]
-                                              derby? postgres?]]
-            [next.jdbc.prepare :as prep]
             [next.jdbc.result-set :as rs]
             [next.jdbc.specs :as specs])
   (:import (java.sql ResultSet ResultSetMetaData)))
@@ -21,16 +18,16 @@
 
 (deftest logging-test
   (let [logging (atom [])
-        logger  (fn [data _] (swap! logging conj data) data)
+        logger  (fn [data _]     (swap! logging conj data)  data)
-
+        log-sql (fn [sql-p opts] (logger sql-p opts) [sql-p opts])
         sql-p   ["select * from fruit where id in (?,?) order by id desc" 1 4]]
     (jdbc/execute! (mw/wrapper (ds))
                    sql-p
                    (assoc (default-options)
-                          :builder-fn rs/as-lower-maps
+                          :builder-fn     rs/as-lower-maps
-                          :sql-params-fn logger
+                          :pre-execute-fn log-sql
-                          :row!-fn logger
+                          :row!-fn        logger
-                          :rs!-fn logger))
+                          :rs!-fn         logger))
     ;; should log four things
     (is (= 4     (-> @logging count)))
     ;; :next.jdbc/sql-params value
@@ -45,9 +42,9 @@
     ;; now repeat without the row logging
     (reset! logging [])
     (jdbc/execute! (mw/wrapper (ds)
-                               {:builder-fn rs/as-lower-maps
+                               {:builder-fn     rs/as-lower-maps
-                                :sql-params-fn logger
+                                :pre-execute-fn log-sql
-                                :rs!-fn logger})
+                                :rs!-fn         logger})
                    sql-p
                    (default-options))
     ;; should log two things
@@ -59,21 +56,22 @@
     (is (= [4 1] (-> @logging (nth 1) (->> (map :fruit/id)))))))
 
 (deftest timing-test
-  (let [timing   (atom {:calls 0 :total 0.0})
+  (let [start-fn (fn [sql-p opts]
-        start-fn (fn [sql-p opts]
+                   (swap! (::timing opts) update ::calls inc)
-                   (swap! (:timing opts) update :calls inc)
+                   [sql-p (assoc opts ::start (System/nanoTime))])
-                   (assoc opts :start (System/nanoTime)))
+        end-fn   (fn [rs opts]
-        exec-fn  (fn [_ opts]
                    (let [end (System/nanoTime)]
-                     (swap! (:timing opts) update :total + (- end (:start opts)))
+                     (swap! (::timing opts) update ::total + (- end (::start opts)))
-                     opts))
+                     [rs opts]))
+        timing   (atom {::calls 0 ::total 0.0})
         sql-p    ["select * from fruit where id in (?,?) order by id desc" 1 4]]
-    (jdbc/execute! (mw/wrapper (ds) {:timing timing
+    (jdbc/execute! (mw/wrapper (ds) {::timing         timing
-                                     :sql-params-fn start-fn
+                                     :pre-execute-fn  start-fn
-                                     :execute-fn    exec-fn})
+                                     :post-execute-fn end-fn})
                    sql-p)
-    (jdbc/execute! (mw/wrapper (ds) {:timing timing
+    (jdbc/execute! (mw/wrapper (ds) {::timing         timing
-                                     :sql-params-fn start-fn
+                                     :pre-execute-fn  start-fn
-                                     :execute-fn    exec-fn})
+                                     :post-execute-fn end-fn})
                    sql-p)
-    (println (db) (:calls @timing) "calls took" (long (:total @timing)) "nanoseconds")))
+    (printf "%20s - %d calls took %,10d nanoseconds\n"
+            (:dbtype (db)) (::calls @timing) (long (::total @timing)))))