From 734c4bfada3b9242496e88ca76eaf62298764b78 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Wed, 22 Jul 2020 13:01:51 -0700 Subject: [PATCH] Fixes #133 by adding :return-generated-keys to execute-batch! --- CHANGELOG.md | 3 +++ doc/all-the-options.md | 3 ++- doc/prepared-statements.md | 16 +++++++++------- src/next/jdbc/prepare.clj | 27 ++++++++++++++++++++++----- test/next/jdbc/prepare_test.clj | 30 ++++++++++++++++++++++++++++-- 5 files changed, 64 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbaa7e6..2c4430a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ Only accretive/fixative changes will be made from now on. +Changes made to **develop** since the 1.1.569 release: +* Address #133 by adding `:return-generated-keys` as an option on `execute-batch!`. + ## Stable Builds * 2020-07-10 -- 1.1.569 diff --git a/doc/all-the-options.md b/doc/all-the-options.md index 4f9fc04..60d3165 100644 --- a/doc/all-the-options.md +++ b/doc/all-the-options.md @@ -90,7 +90,8 @@ Not all databases or drivers support all of these options, or all values for any In addition the the above, `next.jdbc.prepare/execute-batch!` (which does **not** create a `PreparedStatement`) accepts an options hash map that can also contain the following: * `:batch-size` -- an integer that determines how to partition the parameter groups for submitting to the database in batches, -* `:large` -- a Boolean flag that indicates whether the batch will produce large update counts (`long` rather than `int` values). +* `:large` -- a Boolean flag that indicates whether the batch will produce large update counts (`long` rather than `int` values), +* `:return-generated-keys` -- a Boolean flag that indicates whether `.getGeneratedKeys` should be called on the `PreparedStatement` after each batch is executed (if `true`, `execute-batch!` will return a vector of hash maps containing generated keys). ## Transactions diff --git a/doc/prepared-statements.md b/doc/prepared-statements.md index cd5e175..93563fc 100644 --- a/doc/prepared-statements.md +++ b/doc/prepared-statements.md @@ -101,7 +101,9 @@ Both of those are somewhat ugly and contain a fair bit of boilerplate and Java i (p/execute-batch! ps [[1 "Approved"] [2 "Rejected"] [3 "New"]])) ``` -By default, this adds all the parameter groups and executes one batched command. It returns a (Clojure) vector of update counts (rather than `int[]`). If you provide an options hash map, you can specify a `:batch-size` and the parameter groups will be partitioned and executed as multiple batched commands. This is intended to allow very large sequences of parameter groups to be executed without running into limitations that may apply to a single batched command. If you expect the update counts to be very large (more than `Integer/MAX_VALUE`), you can specify `:large true` so that `.executeLargeBatch` is called instead of `.executeBatch`. Note: not all databases support `.executeLargeBatch`. +By default, this adds all the parameter groups and executes one batched command. It returns a (Clojure) vector of update counts (rather than `int[]`). If you provide an options hash map, you can specify a `:batch-size` and the parameter groups will be partitioned and executed as multiple batched commands. This is intended to allow very large sequences of parameter groups to be executed without running into limitations that may apply to a single batched command. If you expect the update counts to be very large (more than `Integer/MAX_VALUE`), you can specify `:large true` so that `.executeLargeBatch` is called instead of `.executeBatch`. + +> Note: not all databases support `.executeLargeBatch`. If you want to get the generated keys from an `insert` done via `execute-batch!`, you need a couple of extras, compared to the above: @@ -110,15 +112,15 @@ If you want to get the generated keys from an `insert` done via `execute-batch!` ;; ensure the PreparedStatement will return the keys: ps (jdbc/prepare con ["insert into status (id,name) values (?,?)"] {:return-keys true})] - ;; this returns update counts (which we'll ignore) - (p/execute-batch! ps [[1 "Approved"] [2 "Rejected"] [3 "New"]]) - ;; this produces the generated keys as a (datafiable) Clojure data structure: - (rs/datafiable-result-set (.getGeneratedKeys ps) con {})) + ;; this will call .getGeneratedKeys for each batch and return them as a + ;; vector of datafiable result sets (the keys in map are database-specific): + (p/execute-batch! ps [[1 "Approved"] [2 "Rejected"] [3 "New"]] + {:return-generated-keys true})) ``` -The call to `rs/datafiable-result-set` can be passed a `:builder-fn` option if you want something other than qualified as-is hash maps. +This calls `rs/datafiable-result-set` behind the scenes so you can also pass a `:builder-fn` option to `execute-batch!` if you want something other than qualified as-is hash maps. -> Note: not all databases support calling `.getGeneratedKeys` here (everything I test against seems to, except MS SQL Server). +> Note: not all databases support calling `.getGeneratedKeys` here (everything I test against seems to, except MS SQL Server). Some databases will only return one generated key per batch, rather than a generated key for every row inserted. ### Caveats diff --git a/src/next/jdbc/prepare.clj b/src/next/jdbc/prepare.clj index 2192b4d..8495bfd 100644 --- a/src/next/jdbc/prepare.clj +++ b/src/next/jdbc/prepare.clj @@ -203,7 +203,15 @@ you can specify `:large true` and `.executeLargeBatch` will be called instead. - Returns a Clojure vector of update counts. + By default, returns a Clojure vector of update counts. Some databases + allow batch statements to also return generated keys and you can attempt that + if you ensure the `PreparedStatement` is created with `:return-keys true` + and you also provide `:return-generated-keys true` in the options passed + to `execute-batch!`. Some databases will only return one generated key + per batch, some return all the generated keys, some will throw an exception. + If that is supported, `execute-batch!` will return a vector of hash maps + containing the generated keys as fully-realized, datafiable result sets, + whose content is database-dependent. May throw `java.sql.BatchUpdateException` if any part of the batch fails. You may be able to call `.getUpdateCounts` on that exception object to @@ -216,7 +224,15 @@ ([ps param-groups] (execute-batch! ps param-groups {})) ([^PreparedStatement ps param-groups opts] - (let [params (if-let [n (:batch-size opts)] + (let [gen-ks (when (:return-generated-keys opts) + (try + (let [drs (requiring-resolve + 'next.jdbc.result-set/datafiable-result-set)] + #(drs (.getGeneratedKeys ^PreparedStatement %) + (p/get-connection ps {}) + opts)) + (catch Throwable _))) + params (if-let [n (:batch-size opts)] (if (and (number? n) (pos? n)) (partition-all n param-groups) (throw (IllegalArgumentException. @@ -225,7 +241,8 @@ (into [] (mapcat (fn [group] (run! #(.addBatch (set-parameters ps %)) group) - (if (:large opts) - (.executeLargeBatch ps) - (.executeBatch ps)))) + (let [result (if (:large opts) + (.executeLargeBatch ps) + (.executeBatch ps))] + (if gen-ks (gen-ks ps) result)))) params)))) diff --git a/test/next/jdbc/prepare_test.clj b/test/next/jdbc/prepare_test.clj index 00780a3..4172eb9 100644 --- a/test/next/jdbc/prepare_test.clj +++ b/test/next/jdbc/prepare_test.clj @@ -9,7 +9,7 @@ (:require [clojure.test :refer [deftest is testing use-fixtures]] [next.jdbc :as jdbc] [next.jdbc.test-fixtures - :refer [with-test-db ds jtds? postgres? sqlite?]] + :refer [with-test-db db ds jtds? mssql? postgres? sqlite?]] [next.jdbc.prepare :as prep] [next.jdbc.specs :as specs])) @@ -74,7 +74,7 @@ INSERT INTO fruit (name, appearance) VALUES (?,?) (conj result (count (jdbc/execute! t ["select * from fruit"])))))))) (is (= 4 (count (jdbc/execute! (ds) ["select * from fruit"]))))) (testing "large batch insert" - (when-not (or (jtds?) (postgres?) (sqlite?)) + (when-not (or (jtds?) (sqlite?)) (is (= [1 1 1 1 1 1 1 1 1 13] (jdbc/with-transaction [t (ds) {:rollback-only true}] (with-open [ps (jdbc/prepare t [" @@ -92,4 +92,30 @@ INSERT INTO fruit (name, appearance) VALUES (?,?) {:batch-size 4 :large true})] (conj result (count (jdbc/execute! t ["select * from fruit"])))))))) + (is (= 4 (count (jdbc/execute! (ds) ["select * from fruit"])))))) + (testing "return generated keys" + (when-not (mssql?) + (let [results + (jdbc/with-transaction [t (ds) {:rollback-only true}] + (with-open [ps (jdbc/prepare t [" +INSERT INTO fruit (name, appearance) VALUES (?,?) +"] + {:return-keys true})] + (let [result (prep/execute-batch! ps [["fruit1" "one"] + ["fruit2" "two"] + ["fruit3" "three"] + ["fruit4" "four"] + ["fruit5" "five"] + ["fruit6" "six"] + ["fruit7" "seven"] + ["fruit8" "eight"] + ["fruit9" "nine"]] + {:batch-size 4 + :return-generated-keys true})] + (conj result (count (jdbc/execute! t ["select * from fruit"]))))))] + (is (= 13 (last results))) + (is (every? map? (butlast results))) + ;; Derby and SQLite only return one generated key per batch so there + ;; are only three keys, plus the overall count here: + (is (< 3 (count results)))) (is (= 4 (count (jdbc/execute! (ds) ["select * from fruit"])))))))