From 3a6539f3e2704ad79a7b070c4ab13d8671adf155 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Fri, 5 Jun 2020 18:34:08 -0700 Subject: [PATCH 01/18] Switch to zonky embedded PG; add jtds testing; add logging control --- deps.edn | 13 +++++-- test/log4j2-jdbc.properties | 13 +++++++ test/next/jdbc/test_fixtures.clj | 57 +++++++++++++++++++++++------ test/next/jdbc/transaction_test.clj | 13 ++++++- test/next/jdbc_test.clj | 21 +++++++++-- 5 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 test/log4j2-jdbc.properties diff --git a/deps.edn b/deps.edn index df5ed63..92893f2 100644 --- a/deps.edn +++ b/deps.edn @@ -19,12 +19,19 @@ org.mariadb.jdbc/mariadb-java-client {:mvn/version "2.5.4"} mysql/mysql-connector-java {:mvn/version "8.0.19"} org.postgresql/postgresql {:mvn/version "42.2.10"} - com.opentable.components/otj-pg-embedded {:mvn/version "0.13.3"} - com.impossibl.pgjdbc-ng/pgjdbc-ng {:mvn/version "0.8.3"} + io.zonky.test/embedded-postgres {:mvn/version "1.2.6"} + io.zonky.test.postgres/embedded-postgres-binaries-darwin-amd64 {:mvn/version "12.2.0"} org.xerial/sqlite-jdbc {:mvn/version "3.30.1"} com.microsoft.sqlserver/mssql-jdbc {:mvn/version "8.2.1.jre8"} ;; supplementary test stuff - org.slf4j/slf4j-nop {:mvn/version "1.7.30"}}} + ;; use log4j 2.x: + org.apache.logging.log4j/log4j-api {:mvn/version "2.13.3"} + ;; bridge into log4j: + org.apache.logging.log4j/log4j-1.2-api {:mvn/version "2.13.3"} + org.apache.logging.log4j/log4j-jcl {:mvn/version "2.13.3"} + org.apache.logging.log4j/log4j-jul {:mvn/version "2.13.3"} + org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.13.3"}} + :jvm-opts ["-Dlog4j2.configurationFile=log4j2-jdbc.properties"]} :runner {:extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner" diff --git a/test/log4j2-jdbc.properties b/test/log4j2-jdbc.properties new file mode 100644 index 0000000..a4e2406 --- /dev/null +++ b/test/log4j2-jdbc.properties @@ -0,0 +1,13 @@ +# Sean's normal mode, shows INFO and above, with highlighting: +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%c] %highlight{%m}%n + +# I do not care about any of c3p0's INFO messages: +logger.c3p0.name = com.mchange.v2 +logger.c3p0.level = warn +logger.c3p0.appenderRef.stdout.ref = STDOUT diff --git a/test/next/jdbc/test_fixtures.clj b/test/next/jdbc/test_fixtures.clj index 7701dad..5b9197c 100644 --- a/test/next/jdbc/test_fixtures.clj +++ b/test/next/jdbc/test_fixtures.clj @@ -5,7 +5,7 @@ (:require [clojure.string :as str] [next.jdbc :as jdbc] [next.jdbc.sql :as sql]) - (:import (com.opentable.db.postgres.embedded EmbeddedPostgres))) + (:import (io.zonky.test.db.postgres.embedded EmbeddedPostgres))) (set! *warn-on-reflection* true) @@ -39,10 +39,16 @@ (def ^:private test-mssql (when (System/getenv "NEXT_JDBC_TEST_MSSQL") test-mssql-map)) +(def ^:private test-jtds-map + {:dbtype "jtds" :dbname "model" + :user "sa" :password (System/getenv "MSSQL_SA_PASSWORD")}) +(def ^:private test-jtds + (when (System/getenv "NEXT_JDBC_TEST_MSSQL") test-jtds-map)) + (def ^:private test-db-specs (cond-> [test-derby test-h2-mem test-h2 test-hsql test-sqlite test-postgres] test-mysql (conj test-mysql) - test-mssql (conj test-mssql))) + test-mssql (conj test-mssql test-jtds))) (def ^:private test-db-spec (atom nil)) @@ -50,7 +56,7 @@ (defn maria? [] (= "mariadb" (:dbtype @test-db-spec))) -(defn mssql? [] (= "mssql" (:dbtype @test-db-spec))) +(defn mssql? [] (#{"jtds" "mssql"} (:dbtype @test-db-spec))) (defn mysql? [] (#{"mariadb" "mysql"} (:dbtype @test-db-spec))) @@ -58,6 +64,8 @@ (defn sqlite? [] (= "sqlite" (:dbtype @test-db-spec))) +(defn stored-proc? [] (not (#{"derby" "h2" "h2:mem" "sqlite"} (:dbtype @test-db-spec)))) + (defn column [k] (let [n (namespace k)] (keyword (when n (cond (postgres?) (str/lower-case n) @@ -113,9 +121,13 @@ :else "AUTO_INCREMENT PRIMARY KEY")] (with-open [con (jdbc/get-connection (ds))] + (when (stored-proc?) + (try + (jdbc/execute-one! con ["DROP PROCEDURE FRUITP"]) + (catch Throwable _))) (try (jdbc/execute-one! con [(str "DROP TABLE " fruit)]) - (catch Exception _)) + (catch Throwable _)) (jdbc/execute-one! con [(str " CREATE TABLE " fruit " ( ID INTEGER " auto-inc-pk ", @@ -124,14 +136,35 @@ CREATE TABLE " fruit " ( COST INT DEFAULT NULL, GRADE REAL DEFAULT NULL )")]) - (sql/insert-multi! con :fruit - [:name :appearance :cost :grade] - [["Apple" "red" 59 nil] - ["Banana" "yellow" nil 92.2] - ["Peach" nil 139 90.0] - ["Orange" "juicy" 89 88.6]] - {:return-keys false}) - (t))))) + (when (stored-proc?) + (let [[begin end] (if (postgres?) ["$$" "$$"] ["BEGIN" "END"])] + (try + (jdbc/execute-one! con [(str " +CREATE PROCEDURE FRUITP" (cond (= "hsqldb" (:dbtype db)) "() READS SQL DATA DYNAMIC RESULT SETS 2 " + (mssql?) " AS " + (postgres?) "() LANGUAGE SQL AS " + :else "() ") " + " begin " " (if (= "hsqldb" (:dbtype db)) + (str "ATOMIC + DECLARE result1 CURSOR WITH RETURN FOR SELECT * FROM " fruit " WHERE COST < 90; + DECLARE result2 CURSOR WITH RETURN FOR SELECT * FROM " fruit " WHERE GRADE >= 90.0; + OPEN result1; + OPEN result2;") + (str " + SELECT * FROM " fruit " WHERE COST < 90; + SELECT * FROM " fruit " WHERE GRADE >= 90.0;")) " + " end " +")]) + (catch Throwable t + (println 'procedure (:dbtype db) (ex-message t)))))) + (sql/insert-multi! con :fruit + [:name :appearance :cost :grade] + [["Apple" "red" 59 nil] + ["Banana" "yellow" nil 92.2] + ["Peach" nil 139 90.0] + ["Orange" "juicy" 89 88.6]] + {:return-keys false}) + (t))))) (comment ;; this is a convenience to bring next.jdbc's test dependencies diff --git a/test/next/jdbc/transaction_test.clj b/test/next/jdbc/transaction_test.clj index 2c2c852..891c8d0 100644 --- a/test/next/jdbc/transaction_test.clj +++ b/test/next/jdbc/transaction_test.clj @@ -2,7 +2,16 @@ (ns next.jdbc.transaction-test "Stub test namespace for transaction handling." - (:require [clojure.test :refer [deftest is testing]] - [next.jdbc.transaction :refer :all])) + (:require [clojure.test :refer [deftest is testing use-fixtures]] + [next.jdbc :as jdbc] + [next.jdbc.specs :as specs] + [next.jdbc.test-fixtures :refer [with-test-db db ds column + default-options + derby? mssql? mysql? postgres?]] + [next.jdbc.transaction :as tx])) (set! *warn-on-reflection* true) + +(use-fixtures :once with-test-db) + +(specs/instrument) diff --git a/test/next/jdbc_test.clj b/test/next/jdbc_test.clj index 65217bd..57151bf 100644 --- a/test/next/jdbc_test.clj +++ b/test/next/jdbc_test.clj @@ -7,7 +7,7 @@ [next.jdbc :as jdbc] [next.jdbc.connection :as c] [next.jdbc.test-fixtures :refer [with-test-db db ds column - default-options + default-options stored-proc? derby? mssql? mysql? postgres?]] [next.jdbc.prepare :as prep] [next.jdbc.result-set :as rs] @@ -302,14 +302,27 @@ VALUES ('Pear', 'green', 49, 47) :else (is (= {} etc))) (is (instance? javax.sql.DataSource ds)) (is (str/index-of (pr-str ds) (str "jdbc:" - (if (mssql?) - "sqlserver" ; mssql is the alias - (:dbtype (db)))))) + (condp = (:dbtype (db)) + "mssql" "sqlserver" + "jtds" "jtds:sqlserver" + (:dbtype (db)))))) ;; checks get-datasource on a DataSource is identity (is (identical? ds (jdbc/get-datasource ds))) (with-open [con (jdbc/get-connection ds {})] (is (instance? java.sql.Connection con))))))) +(deftest multi-rs + (when (stored-proc?) + (testing "stored proc; multiple result sets" + (try + (clojure.pprint/pprint + (jdbc/execute! (ds) [(if (mssql?) + "EXEC FRUITP" + "CALL FRUITP()")])) + (catch Throwable t + (println 'call-proc (:dbtype (db)) (ex-message t) (some-> t (ex-cause) (ex-message)))))))) + + (deftest plan-misuse (let [s (pr-str (jdbc/plan (ds) ["select * from fruit"]))] (is (re-find #"missing reduction" s))) From f5f05ad5371fe35475ac281051421fd16b132da4 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Fri, 5 Jun 2020 19:51:05 -0700 Subject: [PATCH 02/18] Add Linux and Windows (embedded) PostgreSQL 12.2.0 --- deps.edn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deps.edn b/deps.edn index 92893f2..9e0a8ba 100644 --- a/deps.edn +++ b/deps.edn @@ -21,6 +21,8 @@ org.postgresql/postgresql {:mvn/version "42.2.10"} io.zonky.test/embedded-postgres {:mvn/version "1.2.6"} io.zonky.test.postgres/embedded-postgres-binaries-darwin-amd64 {:mvn/version "12.2.0"} + io.zonky.test.postgres/embedded-postgres-binaries-linux-amd64 {:mvn/version "12.2.0"} + io.zonky.test.postgres/embedded-postgres-binaries-windows-amd64 {:mvn/version "12.2.0"} org.xerial/sqlite-jdbc {:mvn/version "3.30.1"} com.microsoft.sqlserver/mssql-jdbc {:mvn/version "8.2.1.jre8"} ;; supplementary test stuff From 0a27e51f3764e62384d76f1f31aaa28769952858 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sat, 6 Jun 2020 09:32:03 -0700 Subject: [PATCH 03/18] First steps to multiple result sets #116 --- src/next/jdbc/result_set.clj | 31 ++++++++++++++++++++++++++++++- test/next/jdbc_test.clj | 6 +++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index 13b7b36..d469392 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -575,6 +575,18 @@ (.getGeneratedKeys stmt) (catch Exception _))))) +(defn- stmt->result-set' + "Given a `PreparedStatement` and options, execute it and return a `ResultSet` + if possible." + ^ResultSet + [^PreparedStatement stmt go opts] + (if go + (.getResultSet stmt) + (when (:return-keys opts) + (try + (.getGeneratedKeys stmt) + (catch Exception _))))) + (defn- reduce-stmt "Execute the `PreparedStatement`, attempt to get either its `ResultSet` or its generated keys (as a `ResultSet`), and reduce that using the supplied @@ -687,9 +699,26 @@ (first sql-params) (rest sql-params) opts)] + (if (:multi-rs opts) + (loop [go (.execute stmt) acc nil rsn 0] + (let [rs (if-let [rs (stmt->result-set' stmt go opts)] + (datafiable-result-set rs this opts) + (let [n (.getUpdateCount stmt)] + (if (= -1 n) + nil + [{:next.jdbc/update-count (.getUpdateCount stmt)}])))] + (if-not rs + acc + (recur (.getMoreResults stmt) + (if acc + (-> acc + (conj {:next.jdbc/result-set rsn}) + (into rs)) + rs) + (inc rsn))))) (if-let [rs (stmt->result-set stmt opts)] (datafiable-result-set rs this opts) - [{:next.jdbc/update-count (.getUpdateCount stmt)}]))) + [{:next.jdbc/update-count (.getUpdateCount stmt)}])))) java.sql.PreparedStatement ;; we can't tell if this PreparedStatement will return generated diff --git a/test/next/jdbc_test.clj b/test/next/jdbc_test.clj index 57151bf..e05645d 100644 --- a/test/next/jdbc_test.clj +++ b/test/next/jdbc_test.clj @@ -315,10 +315,10 @@ VALUES ('Pear', 'green', 49, 47) (when (stored-proc?) (testing "stored proc; multiple result sets" (try + (println "====" (:dbtype (db)) "====") (clojure.pprint/pprint - (jdbc/execute! (ds) [(if (mssql?) - "EXEC FRUITP" - "CALL FRUITP()")])) + (jdbc/execute! (ds) [(if (mssql?) "EXEC FRUITP" "CALL FRUITP()")] + {:multi-rs true})) (catch Throwable t (println 'call-proc (:dbtype (db)) (ex-message t) (some-> t (ex-cause) (ex-message)))))))) From 9240d41ed6bba1c6c5a207afd1077af5becdab86 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sat, 6 Jun 2020 14:29:53 -0700 Subject: [PATCH 04/18] Rename/cleanup log4j2 config --- deps.edn | 2 +- test/log4j2-info.properties | 28 ++++++++++++++++++++++++++++ test/log4j2-jdbc.properties | 13 ------------- 3 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 test/log4j2-info.properties delete mode 100644 test/log4j2-jdbc.properties diff --git a/deps.edn b/deps.edn index 9e0a8ba..6cc599e 100644 --- a/deps.edn +++ b/deps.edn @@ -33,7 +33,7 @@ org.apache.logging.log4j/log4j-jcl {:mvn/version "2.13.3"} org.apache.logging.log4j/log4j-jul {:mvn/version "2.13.3"} org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.13.3"}} - :jvm-opts ["-Dlog4j2.configurationFile=log4j2-jdbc.properties"]} + :jvm-opts ["-Dlog4j2.configurationFile=log4j2-info.properties"]} :runner {:extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner" diff --git a/test/log4j2-info.properties b/test/log4j2-info.properties new file mode 100644 index 0000000..a3bc4d3 --- /dev/null +++ b/test/log4j2-info.properties @@ -0,0 +1,28 @@ +# Sean's normal mode, shows INFO and above, with highlighting: +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%c] %highlight{%m}%n + +# I do not care about any of c3p0's INFO messages... +logger.c3p0.name = com.mchange.v2 +logger.c3p0.level = warn +logger.c3p0.appenderRef.stdout.ref = STDOUT + +# ...nor HikariCP... +logger.hikari.name = com.zaxxer.hikari +logger.hikari.level = warn +logger.hikari.appenderRef.stdout.ref = STDOUT + +# ...nor embedded HSQLDB... +logger.hsqldb.name = hsqldb.db +logger.hsqldb.level = warn +logger.hsqldb.appenderRef.stdout.ref = STDOUT + +# ...nor embedded PostgreSQL... +logger.postgres.name = io.zonky.test.db.postgres.embedded +logger.postgres.level = warn +logger.postgres.appenderRef.stdout.ref = STDOUT diff --git a/test/log4j2-jdbc.properties b/test/log4j2-jdbc.properties deleted file mode 100644 index a4e2406..0000000 --- a/test/log4j2-jdbc.properties +++ /dev/null @@ -1,13 +0,0 @@ -# Sean's normal mode, shows INFO and above, with highlighting: -rootLogger.level = info -rootLogger.appenderRef.stdout.ref = STDOUT - -appender.console.type = Console -appender.console.name = STDOUT -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%c] %highlight{%m}%n - -# I do not care about any of c3p0's INFO messages: -logger.c3p0.name = com.mchange.v2 -logger.c3p0.level = warn -logger.c3p0.appenderRef.stdout.ref = STDOUT From ec31aaa4275b481efc0a1f2046654ecaa9eaa979 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sat, 6 Jun 2020 14:30:27 -0700 Subject: [PATCH 05/18] Add in jTDS tests --- test/next/jdbc/datafy_test.clj | 21 +++++++++++++++------ test/next/jdbc/prepare_test.clj | 5 +++-- test/next/jdbc/sql_test.clj | 3 ++- test/next/jdbc/test_fixtures.clj | 2 ++ 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/test/next/jdbc/datafy_test.clj b/test/next/jdbc/datafy_test.clj index 9be5452..0ed73f1 100644 --- a/test/next/jdbc/datafy_test.clj +++ b/test/next/jdbc/datafy_test.clj @@ -9,8 +9,9 @@ [next.jdbc.datafy] [next.jdbc.result-set :as rs] [next.jdbc.specs :as specs] - [next.jdbc.test-fixtures :refer [with-test-db db ds - derby? mysql? postgres? sqlite?]])) + [next.jdbc.test-fixtures + :refer [with-test-db db ds + derby? jtds? mysql? postgres? sqlite?]])) (set! *warn-on-reflection* true) @@ -32,7 +33,11 @@ (with-open [con (jdbc/get-connection (ds))] (let [reference-keys (cond-> basic-connection-keys (derby?) (-> (disj :networkTimeout) - (conj :networkTimeout/exception))) + (conj :networkTimeout/exception)) + (jtds?) (-> (disj :clientInfo :networkTimeout :schema) + (conj :clientInfo/exception + :networkTimeout/exception + :schema/exception))) data (set (keys (d/datafy con)))] (when-let [diff (seq (set/difference data reference-keys))] (println (:dbtype (db)) :connection (sort diff))) @@ -72,11 +77,14 @@ (testing "database metadata datafication" (with-open [con (jdbc/get-connection (ds))] (let [reference-keys (cond-> basic-database-metadata-keys + (jtds?) (-> (disj :clientInfoProperties :rowIdLifetime) + (conj :clientInfoProperties/exception + :rowIdLifetime/exception)) (postgres?) (-> (disj :rowIdLifetime) (conj :rowIdLifetime/exception)) - (sqlite?) (-> (disj :clientInfoProperties :rowIdLifetime) - (conj :clientInfoProperties/exception - :rowIdLifetime/exception))) + (sqlite?) (-> (disj :clientInfoProperties :rowIdLifetime) + (conj :clientInfoProperties/exception + :rowIdLifetime/exception))) data (set (keys (d/datafy (.getMetaData con))))] (when-let [diff (seq (set/difference data reference-keys))] (println (:dbtype (db)) :db-meta (sort diff))) @@ -86,6 +94,7 @@ (with-open [con (jdbc/get-connection (ds))] (let [data (d/datafy (.getMetaData con))] (doseq [k (cond-> #{:catalogs :clientInfoProperties :schemas :tableTypes :typeInfo} + (jtds?) (disj :clientInfoProperties) (sqlite?) (disj :clientInfoProperties))] (let [rs (d/nav data k nil)] (is (vector? rs)) diff --git a/test/next/jdbc/prepare_test.clj b/test/next/jdbc/prepare_test.clj index 04e71c0..00780a3 100644 --- a/test/next/jdbc/prepare_test.clj +++ b/test/next/jdbc/prepare_test.clj @@ -8,7 +8,8 @@ `execute-batch!` here." (:require [clojure.test :refer [deftest is testing use-fixtures]] [next.jdbc :as jdbc] - [next.jdbc.test-fixtures :refer [with-test-db ds postgres? sqlite?]] + [next.jdbc.test-fixtures + :refer [with-test-db ds jtds? postgres? sqlite?]] [next.jdbc.prepare :as prep] [next.jdbc.specs :as specs])) @@ -73,7 +74,7 @@ INSERT INTO fruit (name, appearance) VALUES (?,?) (conj result (count (jdbc/execute! t ["select * from fruit"])))))))) (is (= 4 (count (jdbc/execute! (ds) ["select * from fruit"]))))) (testing "large batch insert" - (when-not (or (postgres?) (sqlite?)) + (when-not (or (jtds?) (postgres?) (sqlite?)) (is (= [1 1 1 1 1 1 1 1 1 13] (jdbc/with-transaction [t (ds) {:rollback-only true}] (with-open [ps (jdbc/prepare t [" diff --git a/test/next/jdbc/sql_test.clj b/test/next/jdbc/sql_test.clj index fd21a60..15cae96 100644 --- a/test/next/jdbc/sql_test.clj +++ b/test/next/jdbc/sql_test.clj @@ -7,7 +7,7 @@ [next.jdbc.sql :as sql] [next.jdbc.test-fixtures :refer [with-test-db ds column default-options - derby? maria? mssql? mysql? postgres? sqlite?]])) + derby? jtds? maria? mssql? mysql? postgres? sqlite?]])) (set! *warn-on-reflection* true) @@ -67,6 +67,7 @@ (deftest test-insert-delete (let [new-key (cond (derby?) :1 + (jtds?) :ID (maria?) :insert_id (mssql?) :GENERATED_KEYS (mysql?) :GENERATED_KEY diff --git a/test/next/jdbc/test_fixtures.clj b/test/next/jdbc/test_fixtures.clj index 5b9197c..3d78750 100644 --- a/test/next/jdbc/test_fixtures.clj +++ b/test/next/jdbc/test_fixtures.clj @@ -54,6 +54,8 @@ (defn derby? [] (= "derby" (:dbtype @test-db-spec))) +(defn jtds? [] (= "jtds" (:dbtype @test-db-spec))) + (defn maria? [] (= "mariadb" (:dbtype @test-db-spec))) (defn mssql? [] (#{"jtds" "mssql"} (:dbtype @test-db-spec))) From 326977dddce9526c7373fdea29b4b1cc593bee7a Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sat, 6 Jun 2020 14:38:09 -0700 Subject: [PATCH 06/18] :multi-rs truthy defaults to sequnce; :delimited available #116 --- src/next/jdbc/result_set.clj | 17 ++++++++++------- test/next/jdbc_test.clj | 6 +++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index d469392..b821bdc 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -699,8 +699,8 @@ (first sql-params) (rest sql-params) opts)] - (if (:multi-rs opts) - (loop [go (.execute stmt) acc nil rsn 0] + (if-let [multi (:multi-rs opts)] + (loop [go (.execute stmt) acc (if (= :delimited multi) nil []) rsn 0] (let [rs (if-let [rs (stmt->result-set' stmt go opts)] (datafiable-result-set rs this opts) (let [n (.getUpdateCount stmt)] @@ -710,11 +710,14 @@ (if-not rs acc (recur (.getMoreResults stmt) - (if acc - (-> acc - (conj {:next.jdbc/result-set rsn}) - (into rs)) - rs) + (cond (not= :delimited multi) + (conj acc rs) + acc + (-> acc + (conj {:next.jdbc/result-set rsn}) + (into rs)) + :else + rs) (inc rsn))))) (if-let [rs (stmt->result-set stmt opts)] (datafiable-result-set rs this opts) diff --git a/test/next/jdbc_test.clj b/test/next/jdbc_test.clj index e05645d..4d601a9 100644 --- a/test/next/jdbc_test.clj +++ b/test/next/jdbc_test.clj @@ -315,10 +315,14 @@ VALUES ('Pear', 'green', 49, 47) (when (stored-proc?) (testing "stored proc; multiple result sets" (try - (println "====" (:dbtype (db)) "====") + (println "====" (:dbtype (db)) "==== true") (clojure.pprint/pprint (jdbc/execute! (ds) [(if (mssql?) "EXEC FRUITP" "CALL FRUITP()")] {:multi-rs true})) + (println "====" (:dbtype (db)) "==== :delimited") + (clojure.pprint/pprint + (jdbc/execute! (ds) [(if (mssql?) "EXEC FRUITP" "CALL FRUITP()")] + {:multi-rs :delimited})) (catch Throwable t (println 'call-proc (:dbtype (db)) (ex-message t) (some-> t (ex-cause) (ex-message)))))))) From 067919a2963fe99bc766cd143b765ac1f93cb96f Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sun, 7 Jun 2020 12:13:32 -0700 Subject: [PATCH 07/18] Roll multi-rs across execute-all for all types --- src/next/jdbc/result_set.clj | 105 ++++++++++++++++--------------- test/next/jdbc/test_fixtures.clj | 2 +- test/next/jdbc_test.clj | 6 +- 3 files changed, 57 insertions(+), 56 deletions(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index be0f715..1150dea 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -582,17 +582,25 @@ (.getGeneratedKeys stmt) (catch Exception _))))) -(defn- stmt->result-set' - "Given a `PreparedStatement` and options, execute it and return a `ResultSet` - if possible." - ^ResultSet - [^PreparedStatement stmt go opts] - (if go - (.getResultSet stmt) - (when (:return-keys opts) - (try - (.getGeneratedKeys stmt) - (catch Exception _))))) +(defn- stmt->result-set-update-count + "Given a connectable, a `Statement`, a flag indicating there might + be a result set, and options, return a (datafiable) result set if possible + (either from the statement or from generated keys). If no result set is + available, return a 'fake' result set containing the update count. + + If no update count is available either, return nil." + [connectable ^Statement stmt go opts] + (if-let [^ResultSet + rs (if go + (.getResultSet stmt) + (when (:return-keys opts) + (try + (.getGeneratedKeys stmt) + (catch Exception _))))] + (datafiable-result-set rs connectable opts) + (let [n (.getUpdateCount stmt)] + (when-not (= -1 n) + [{:next.jdbc/update-count n}])))) (defn- reduce-stmt "Execute the `PreparedStatement`, attempt to get either its `ResultSet` or @@ -616,15 +624,14 @@ (defn- stmt-sql->result-set "Given a `Statement`, a SQL command, and options, execute it and return a - `ResultSet` if possible." + `ResultSet` if possible. We always attempt to return keys." ^ResultSet [^Statement stmt ^String sql opts] (if (.execute stmt sql) (.getResultSet stmt) - (when (:return-keys opts) - (try - (.getGeneratedKeys stmt) - (catch Exception _))))) + (try + (.getGeneratedKeys stmt) + (catch Exception _)))) (defn- reduce-stmt-sql "Execute the SQL command on the given `Statement`, attempt to get either @@ -673,9 +680,14 @@ (first sql-params) (rest sql-params) opts)] - (if-let [rs (stmt->result-set stmt opts)] - (datafiable-result-set rs this opts) - [{:next.jdbc/update-count (.getUpdateCount stmt)}]))) + (if (:multi-rs opts) + (loop [go (.execute stmt) acc [] rsn 0] + (if-let [rs (stmt->result-set-update-count this stmt go opts)] + (recur (.getMoreResults stmt) (conj acc rs) (inc rsn)) + acc)) + (if-let [rs (stmt->result-set stmt opts)] + (datafiable-result-set rs this opts) + [{:next.jdbc/update-count (.getUpdateCount stmt)}])))) javax.sql.DataSource (-execute [this sql-params opts] @@ -706,26 +718,11 @@ (first sql-params) (rest sql-params) opts)] - (if-let [multi (:multi-rs opts)] - (loop [go (.execute stmt) acc (if (= :delimited multi) nil []) rsn 0] - (let [rs (if-let [rs (stmt->result-set' stmt go opts)] - (datafiable-result-set rs this opts) - (let [n (.getUpdateCount stmt)] - (if (= -1 n) - nil - [{:next.jdbc/update-count (.getUpdateCount stmt)}])))] - (if-not rs - acc - (recur (.getMoreResults stmt) - (cond (not= :delimited multi) - (conj acc rs) - acc - (-> acc - (conj {:next.jdbc/result-set rsn}) - (into rs)) - :else - rs) - (inc rsn))))) + (if (:multi-rs opts) + (loop [go (.execute stmt) acc [] rsn 0] + (if-let [rs (stmt->result-set-update-count this stmt go opts)] + (recur (.getMoreResults stmt) (conj acc rs) (inc rsn)) + acc)) (if-let [rs (stmt->result-set stmt opts)] (datafiable-result-set rs this opts) [{:next.jdbc/update-count (.getUpdateCount stmt)}])))) @@ -748,25 +745,27 @@ (.getConnection this) opts))) {:next.jdbc/update-count (.getUpdateCount this)})) (-execute-all [this _ opts] - (if-let [rs (stmt->result-set this opts)] - (datafiable-result-set rs (.getConnection this) opts) - [{:next.jdbc/update-count (.getUpdateCount this)}])) + (if (:multi-rs opts) + (loop [go (.execute this) acc [] rsn 0] + (if-let [rs (stmt->result-set-update-count (.getConnection this) this go opts)] + (recur (.getMoreResults this) (conj acc rs) (inc rsn)) + acc)) + (if-let [rs (stmt->result-set this opts)] + (datafiable-result-set rs (.getConnection this) opts) + [{:next.jdbc/update-count (.getUpdateCount this)}]))) java.sql.Statement - ;; we can't tell if this Statement will return generated - ;; keys so we pass a truthy value to at least attempt it if we - ;; do not get a ResultSet back from the execute call (-execute [this sql-params opts] (assert (= 1 (count sql-params)) "Parameters cannot be provided when executing a non-prepared Statement") (reify clojure.lang.IReduceInit (reduce [_ f init] - (reduce-stmt-sql this (first sql-params) f init (assoc opts :return-keys true))) + (reduce-stmt-sql this (first sql-params) f init opts)) (toString [_] "`IReduceInit` from `plan` -- missing reduction?"))) (-execute-one [this sql-params opts] (assert (= 1 (count sql-params)) "Parameters cannot be provided when executing a non-prepared Statement") - (if-let [rs (stmt-sql->result-set this (first sql-params) (assoc opts :return-keys true))] + (if-let [rs (stmt-sql->result-set this (first sql-params) opts)] (let [builder-fn (get opts :builder-fn as-maps) builder (builder-fn rs opts)] (when (.next rs) @@ -776,9 +775,15 @@ (-execute-all [this sql-params opts] (assert (= 1 (count sql-params)) "Parameters cannot be provided when executing a non-prepared Statement") - (if-let [rs (stmt-sql->result-set this (first sql-params) opts)] - (datafiable-result-set rs (.getConnection this) opts) - [{:next.jdbc/update-count (.getUpdateCount this)}])) + (if (:multi-rs opts) + (loop [go (.execute this (first sql-params)) acc [] rsn 0] + (if-let [rs (stmt->result-set-update-count + (.getConnection this) this go (assoc opts :return-keys true))] + (recur (.getMoreResults this) (conj acc rs) (inc rsn)) + acc)) + (if-let [rs (stmt-sql->result-set this (first sql-params) opts)] + (datafiable-result-set rs (.getConnection this) opts) + [{:next.jdbc/update-count (.getUpdateCount this)}]))) Object (-execute [this sql-params opts] diff --git a/test/next/jdbc/test_fixtures.clj b/test/next/jdbc/test_fixtures.clj index ea434ae..e41a2ac 100644 --- a/test/next/jdbc/test_fixtures.clj +++ b/test/next/jdbc/test_fixtures.clj @@ -152,7 +152,7 @@ CREATE TABLE " fruit " ( (when (stored-proc?) (let [[begin end] (if (postgres?) ["$$" "$$"] ["BEGIN" "END"])] (try - (jdbc/execute-one! con [(str " + (do-commands con [(str " CREATE PROCEDURE FRUITP" (cond (= "hsqldb" (:dbtype db)) "() READS SQL DATA DYNAMIC RESULT SETS 2 " (mssql?) " AS " (postgres?) "() LANGUAGE SQL AS " diff --git a/test/next/jdbc_test.clj b/test/next/jdbc_test.clj index 24e4e1c..9d9a1a2 100644 --- a/test/next/jdbc_test.clj +++ b/test/next/jdbc_test.clj @@ -317,14 +317,10 @@ VALUES ('Pear', 'green', 49, 47) (when (stored-proc?) (testing "stored proc; multiple result sets" (try - (println "====" (:dbtype (db)) "==== true") + (println "====" (:dbtype (db)) "====") (clojure.pprint/pprint (jdbc/execute! (ds) [(if (mssql?) "EXEC FRUITP" "CALL FRUITP()")] {:multi-rs true})) - (println "====" (:dbtype (db)) "==== :delimited") - (clojure.pprint/pprint - (jdbc/execute! (ds) [(if (mssql?) "EXEC FRUITP" "CALL FRUITP()")] - {:multi-rs :delimited})) (catch Throwable t (println 'call-proc (:dbtype (db)) (ex-message t) (some-> t (ex-cause) (ex-message)))))))) From a56c18d53165eed9af709ddc68998dbd49e38f71 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sun, 7 Jun 2020 12:35:03 -0700 Subject: [PATCH 08/18] Clean up stmt-sql and :return-keys handling --- src/next/jdbc/result_set.clj | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index 1150dea..abb1119 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -623,10 +623,10 @@ (f init {:next.jdbc/update-count (.getUpdateCount stmt)}))) (defn- stmt-sql->result-set - "Given a `Statement`, a SQL command, and options, execute it and return a + "Given a `Statement`, a SQL command, execute it and return a `ResultSet` if possible. We always attempt to return keys." ^ResultSet - [^Statement stmt ^String sql opts] + [^Statement stmt ^String sql] (if (.execute stmt sql) (.getResultSet stmt) (try @@ -642,7 +642,7 @@ a hash map containing `:next.jdbc/update-count` and the number of rows updated, with the supplied function and initial value applied." [^Statement stmt sql f init opts] - (if-let [rs (stmt-sql->result-set stmt sql opts)] + (if-let [rs (stmt-sql->result-set stmt sql)] (let [rs-map (mapify-result-set rs opts)] (loop [init' init] (if (.next rs) @@ -747,14 +747,18 @@ (-execute-all [this _ opts] (if (:multi-rs opts) (loop [go (.execute this) acc [] rsn 0] - (if-let [rs (stmt->result-set-update-count (.getConnection this) this go opts)] + (if-let [rs (stmt->result-set-update-count + (.getConnection this) this go (assoc opts :return-keys true))] (recur (.getMoreResults this) (conj acc rs) (inc rsn)) acc)) - (if-let [rs (stmt->result-set this opts)] + (if-let [rs (stmt->result-set this (assoc opts :return-keys true))] (datafiable-result-set rs (.getConnection this) opts) [{:next.jdbc/update-count (.getUpdateCount this)}]))) java.sql.Statement + ;; we can't tell if this Statement will return generated keys + ;; but the stmt-sql routines pass a truthy value to at least + ;; attempt it -- so we must explicitly pass it to the other calls (-execute [this sql-params opts] (assert (= 1 (count sql-params)) "Parameters cannot be provided when executing a non-prepared Statement") @@ -765,7 +769,7 @@ (-execute-one [this sql-params opts] (assert (= 1 (count sql-params)) "Parameters cannot be provided when executing a non-prepared Statement") - (if-let [rs (stmt-sql->result-set this (first sql-params) opts)] + (if-let [rs (stmt-sql->result-set this (first sql-params))] (let [builder-fn (get opts :builder-fn as-maps) builder (builder-fn rs opts)] (when (.next rs) @@ -781,7 +785,7 @@ (.getConnection this) this go (assoc opts :return-keys true))] (recur (.getMoreResults this) (conj acc rs) (inc rsn)) acc)) - (if-let [rs (stmt-sql->result-set this (first sql-params) opts)] + (if-let [rs (stmt-sql->result-set this (first sql-params))] (datafiable-result-set rs (.getConnection this) opts) [{:next.jdbc/update-count (.getUpdateCount this)}]))) From 2b0515f6577116a35fde67c1d05e9f6f10c55e9c Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Mon, 22 Jun 2020 23:25:35 -0700 Subject: [PATCH 09/18] Baseline to 1.0.476 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59c9fed..c3ef004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Only accretive/fixative changes will be made from now on. -Changes made on master since 1.0.475: +Changes made on master since 1.0.476: * WIP: support for stored procedures and multiple result sets! ## Stable Builds From 9d6e7ab145a890ebaf65c2a06b75f5475eeebc90 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Wed, 24 Jun 2020 12:25:25 -0700 Subject: [PATCH 10/18] Remove unused rsn Since I changed the default multi-result-set format, the result set number is no longer needed. --- src/next/jdbc/result_set.clj | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index abb1119..4f4f326 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -681,9 +681,9 @@ (rest sql-params) opts)] (if (:multi-rs opts) - (loop [go (.execute stmt) acc [] rsn 0] + (loop [go (.execute stmt) acc []] (if-let [rs (stmt->result-set-update-count this stmt go opts)] - (recur (.getMoreResults stmt) (conj acc rs) (inc rsn)) + (recur (.getMoreResults stmt) (conj acc rs)) acc)) (if-let [rs (stmt->result-set stmt opts)] (datafiable-result-set rs this opts) @@ -719,9 +719,9 @@ (rest sql-params) opts)] (if (:multi-rs opts) - (loop [go (.execute stmt) acc [] rsn 0] + (loop [go (.execute stmt) acc []] (if-let [rs (stmt->result-set-update-count this stmt go opts)] - (recur (.getMoreResults stmt) (conj acc rs) (inc rsn)) + (recur (.getMoreResults stmt) (conj acc rs)) acc)) (if-let [rs (stmt->result-set stmt opts)] (datafiable-result-set rs this opts) @@ -746,10 +746,10 @@ {:next.jdbc/update-count (.getUpdateCount this)})) (-execute-all [this _ opts] (if (:multi-rs opts) - (loop [go (.execute this) acc [] rsn 0] + (loop [go (.execute this) acc []] (if-let [rs (stmt->result-set-update-count (.getConnection this) this go (assoc opts :return-keys true))] - (recur (.getMoreResults this) (conj acc rs) (inc rsn)) + (recur (.getMoreResults this) (conj acc rs)) acc)) (if-let [rs (stmt->result-set this (assoc opts :return-keys true))] (datafiable-result-set rs (.getConnection this) opts) @@ -780,10 +780,10 @@ (assert (= 1 (count sql-params)) "Parameters cannot be provided when executing a non-prepared Statement") (if (:multi-rs opts) - (loop [go (.execute this (first sql-params)) acc [] rsn 0] + (loop [go (.execute this (first sql-params)) acc []] (if-let [rs (stmt->result-set-update-count (.getConnection this) this go (assoc opts :return-keys true))] - (recur (.getMoreResults this) (conj acc rs) (inc rsn)) + (recur (.getMoreResults this) (conj acc rs)) acc)) (if-let [rs (stmt-sql->result-set this (first sql-params))] (datafiable-result-set rs (.getConnection this) opts) From 0cfbb58b2ebf7e8416d8462d77e02318122fffc0 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Wed, 24 Jun 2020 12:33:32 -0700 Subject: [PATCH 11/18] Clean up multi-rs results Accept that PostgreSQL does not support this yet. --- test/next/jdbc/test_fixtures.clj | 8 +++++--- test/next/jdbc_test.clj | 30 +++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/test/next/jdbc/test_fixtures.clj b/test/next/jdbc/test_fixtures.clj index 3d65724..042de4f 100644 --- a/test/next/jdbc/test_fixtures.clj +++ b/test/next/jdbc/test_fixtures.clj @@ -55,6 +55,8 @@ (defn derby? [] (= "derby" (:dbtype @test-db-spec))) +(defn hsqldb? [] (= "hsqldb" (:dbtype @test-db-spec))) + (defn jtds? [] (= "jtds" (:dbtype @test-db-spec))) (defn maria? [] (= "mariadb" (:dbtype @test-db-spec))) @@ -120,7 +122,7 @@ (reset! test-datasource (jdbc/get-datasource db))) (let [fruit (if (mysql?) "fruit" "FRUIT") ; MySQL is case sensitive! auto-inc-pk - (cond (or (derby?) (= "hsqldb" (:dbtype db))) + (cond (or (derby?) (hsqldb?)) (str "GENERATED ALWAYS AS IDENTITY" " (START WITH 1, INCREMENT BY 1)" " PRIMARY KEY") @@ -165,11 +167,11 @@ CREATE TABLE " fruit " ( (let [[begin end] (if (postgres?) ["$$" "$$"] ["BEGIN" "END"])] (try (do-commands con [(str " -CREATE PROCEDURE FRUITP" (cond (= "hsqldb" (:dbtype db)) "() READS SQL DATA DYNAMIC RESULT SETS 2 " +CREATE PROCEDURE FRUITP" (cond (hsqldb?) "() READS SQL DATA DYNAMIC RESULT SETS 2 " (mssql?) " AS " (postgres?) "() LANGUAGE SQL AS " :else "() ") " - " begin " " (if (= "hsqldb" (:dbtype db)) + " begin " " (if (hsqldb?) (str "ATOMIC DECLARE result1 CURSOR WITH RETURN FOR SELECT * FROM " fruit " WHERE COST < 90; DECLARE result2 CURSOR WITH RETURN FOR SELECT * FROM " fruit " WHERE GRADE >= 90.0; diff --git a/test/next/jdbc_test.clj b/test/next/jdbc_test.clj index b94b863..079da42 100644 --- a/test/next/jdbc_test.clj +++ b/test/next/jdbc_test.clj @@ -6,9 +6,10 @@ [clojure.test :refer [deftest is testing use-fixtures]] [next.jdbc :as jdbc] [next.jdbc.connection :as c] - [next.jdbc.test-fixtures :refer [with-test-db db ds column - default-options stored-proc? - derby? jtds? mssql? mysql? postgres?]] + [next.jdbc.test-fixtures + :refer [with-test-db db ds column + default-options stored-proc? + derby? hsqldb? jtds? mssql? mysql? postgres?]] [next.jdbc.prepare :as prep] [next.jdbc.result-set :as rs] [next.jdbc.specs :as specs]) @@ -320,10 +321,25 @@ VALUES ('Pear', 'green', 49, 47) (when (stored-proc?) (testing "stored proc; multiple result sets" (try - (println "====" (:dbtype (db)) "====") - (clojure.pprint/pprint - (jdbc/execute! (ds) [(if (mssql?) "EXEC FRUITP" "CALL FRUITP()")] - {:multi-rs true})) + (let [multi-rs + (jdbc/execute! (ds) + [(if (mssql?) "EXEC FRUITP" "CALL FRUITP()")] + {:multi-rs true}) + zero-updates [{:next.jdbc/update-count 0}]] + (cond (postgres?) ; does not support multiple result sets yet + (do + (is (= 1 (count multi-rs))) + (is (= zero-updates (first multi-rs)))) + (hsqldb?) + (do + (is (= 3 (count multi-rs))) + (is (= zero-updates (first multi-rs)))) + (mysql?) + (do + (is (= 3 (count multi-rs))) + (is (= zero-updates (last multi-rs)))) + :else + (is (= 2 (count multi-rs))))) (catch Throwable t (println 'call-proc (:dbtype (db)) (ex-message t) (some-> t (ex-cause) (ex-message)))))))) From 25cdb3bc3b680272a7cac5ec1c8e3f49d1bc390a Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Fri, 26 Jun 2020 15:38:35 -0700 Subject: [PATCH 12/18] Add test of begin/end TSQL script with multiple selects --- test/next/jdbc_test.clj | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/next/jdbc_test.clj b/test/next/jdbc_test.clj index 079da42..a133183 100644 --- a/test/next/jdbc_test.clj +++ b/test/next/jdbc_test.clj @@ -318,6 +318,18 @@ VALUES ('Pear', 'green', 49, 47) (is (instance? java.sql.Connection con))))))) (deftest multi-rs + (when (mssql?) + (testing "script with multiple result sets" + (let [multi-rs + (jdbc/execute! (ds) + [(str "begin" + " select * from fruit;" + " select * from fruit where id < 4;" + " end")] + {:multi-rs true})] + (is (= 2 (count multi-rs))) + (is (= 4 (count (first multi-rs)))) + (is (= 3 (count (second multi-rs))))))) (when (stored-proc?) (testing "stored proc; multiple result sets" (try From 217621cb1cea2ff680f0b875b6db5b1fd9ee0edf Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Fri, 26 Jun 2020 22:43:14 -0700 Subject: [PATCH 13/18] Fix fold statement --- src/next/jdbc/result_set.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index b53d6d1..2a2edfc 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -698,7 +698,7 @@ a hash map containing `:next.jdbc/update-count` and the number of rows updated, and fold that as a single element collection." [^Statement stmt sql n combinef reducef connectable opts] - (if-let [rs (stmt-sql->result-set stmt sql opts)] + (if-let [rs (stmt-sql->result-set stmt sql)] (let [rs-map (mapify-result-set rs opts) chunk (fn [batch] (#'r/fjtask #(r/reduce reducef (combinef) batch))) realize (fn [row] (datafiable-row row connectable opts))] From eb5bfef5854411e0d372b40842c58319768d077c Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Fri, 26 Jun 2020 22:47:44 -0700 Subject: [PATCH 14/18] Restore opts/:return-keys optimization for stmt-sql --- src/next/jdbc/result_set.clj | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index 2a2edfc..321e63d 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -662,12 +662,13 @@ "Given a `Statement`, a SQL command, execute it and return a `ResultSet` if possible. We always attempt to return keys." ^ResultSet - [^Statement stmt ^String sql] + [^Statement stmt ^String sql opts] (if (.execute stmt sql) (.getResultSet stmt) - (try - (.getGeneratedKeys stmt) - (catch Exception _)))) + (when (:return-keys opts) + (try + (.getGeneratedKeys stmt) + (catch Exception _))))) (defn- reduce-stmt-sql "Execute the SQL command on the given `Statement`, attempt to get either @@ -678,7 +679,7 @@ a hash map containing `:next.jdbc/update-count` and the number of rows updated, with the supplied function and initial value applied." [^Statement stmt sql f init opts] - (if-let [rs (stmt-sql->result-set stmt sql)] + (if-let [rs (stmt-sql->result-set stmt sql opts)] (let [rs-map (mapify-result-set rs opts)] (loop [init' init] (if (.next rs) @@ -698,7 +699,7 @@ a hash map containing `:next.jdbc/update-count` and the number of rows updated, and fold that as a single element collection." [^Statement stmt sql n combinef reducef connectable opts] - (if-let [rs (stmt-sql->result-set stmt sql)] + (if-let [rs (stmt-sql->result-set stmt sql opts)] (let [rs-map (mapify-result-set rs opts) chunk (fn [batch] (#'r/fjtask #(r/reduce reducef (combinef) batch))) realize (fn [row] (datafiable-row row connectable opts))] From d6693a90323bdc5598685ad2f01cebda690b5103 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Fri, 26 Jun 2020 22:56:33 -0700 Subject: [PATCH 15/18] More stmt-sql bug fixing --- src/next/jdbc/result_set.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index 321e63d..cb42b0d 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -870,7 +870,8 @@ (-execute-one [this sql-params opts] (assert (= 1 (count sql-params)) "Parameters cannot be provided when executing a non-prepared Statement") - (if-let [rs (stmt-sql->result-set this (first sql-params))] + (if-let [rs (stmt-sql->result-set this (first sql-params) + (assoc opts :return-keys true))] (let [builder-fn (get opts :builder-fn as-maps) builder (builder-fn rs opts)] (when (.next rs) @@ -886,7 +887,8 @@ (.getConnection this) this go (assoc opts :return-keys true))] (recur (.getMoreResults this) (conj acc rs)) acc)) - (if-let [rs (stmt-sql->result-set this (first sql-params))] + (if-let [rs (stmt-sql->result-set this (first sql-params) + (assoc opts :return-keys true))] (datafiable-result-set rs (.getConnection this) opts) [{:next.jdbc/update-count (.getUpdateCount this)}]))) From bb0447a14ee8813a04245e9647542b871a532237 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Fri, 26 Jun 2020 23:15:03 -0700 Subject: [PATCH 16/18] Final optimization on stmt-sql return-keys --- src/next/jdbc/result_set.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/next/jdbc/result_set.clj b/src/next/jdbc/result_set.clj index 883d5c2..194a9fe 100644 --- a/src/next/jdbc/result_set.clj +++ b/src/next/jdbc/result_set.clj @@ -880,8 +880,7 @@ (.getConnection this) this go (assoc opts :return-keys true))] (recur (.getMoreResults this) (conj acc rs)) acc)) - (if-let [rs (stmt-sql->result-set this (first sql-params) - (assoc opts :return-keys true))] + (if-let [rs (stmt-sql->result-set this (first sql-params))] (datafiable-result-set rs (.getConnection this) opts) [{:next.jdbc/update-count (.getUpdateCount this)}]))) From 112f9622bd08cd91de8cfb46950ae036162704b4 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sat, 27 Jun 2020 15:06:19 -0700 Subject: [PATCH 17/18] Clean up CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de5d3fb..ef7e2cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ Only accretive/fixative changes will be made from now on. -Changes made since the 1.0.478 release: * WIP support multiple result sets. + +Changes made since the 1.0.478 release: * Address #125 by making the result of `plan` foldable (in the `clojure.core.reducers` sense). * Address #124 by extending `next.jdbc.sql.builder/for-query` to support `:top` (SQL Server), `:limit` / `:offset` (MySQL/PostgreSQL), `:offset` / `:fetch` (SQL Standard). * Allow `:all` to be passed into `find-by-keys` instead of an example hash map or a where clause vector so all rows will be returned (expected to be used with `:offset` etc to support simple pagination of an entire table). From c09d72f1873a71deba1faa7285e5d69416e609c7 Mon Sep 17 00:00:00 2001 From: Sean Corfield Date: Sat, 27 Jun 2020 16:10:23 -0700 Subject: [PATCH 18/18] Fixes #116 by documenting multiple result sets --- CHANGELOG.md | 3 +-- doc/all-the-options.md | 2 ++ doc/getting-started.md | 2 ++ src/next/jdbc.clj | 5 ++++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef7e2cf..e177481 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,10 @@ Only accretive/fixative changes will be made from now on. -* WIP support multiple result sets. - Changes made since the 1.0.478 release: * Address #125 by making the result of `plan` foldable (in the `clojure.core.reducers` sense). * Address #124 by extending `next.jdbc.sql.builder/for-query` to support `:top` (SQL Server), `:limit` / `:offset` (MySQL/PostgreSQL), `:offset` / `:fetch` (SQL Standard). +* Address #116 by adding a `:multi-rs` option to `execute!` to retrieve multiple result sets, for example from stored procedure calls or T-SQL scripts. * Allow `:all` to be passed into `find-by-keys` instead of an example hash map or a where clause vector so all rows will be returned (expected to be used with `:offset` etc to support simple pagination of an entire table). * Add `:columns` option to `find-by-keys` (and `get-by-id`) to specify a subset of columns to be returned in each row. This can also specify an alias for the column and allows for computed expressions to be selected with an alias. diff --git a/doc/all-the-options.md b/doc/all-the-options.md index e9608fe..e233df2 100644 --- a/doc/all-the-options.md +++ b/doc/all-the-options.md @@ -59,6 +59,8 @@ Any function that might realize a row or a result set will accept: * `:label-fn` -- if `:builder-fn` is specified as one of `next.jdbc.result-set`'s `as-modified-*` builders, this option must be present and should specify a string-to-string transformation that will be applied to the column label for each returned column name. * `:qualifier-fn` -- if `:builder-fn` is specified as one of `next.jdbc.result-set`'s `as-modified-*` builders, this option should specify a string-to-string transformation that will be applied to the table name for each returned column name. It will be called with an empty string if the table name is not available. It can be omitted for the `as-unqualified-modified-*` variants. +In addition, `execute!` accepts the `:multi-rs true` option to return multiple result sets -- as a vector of result sets. + > Note: Subject to the caveats above about `:builder-fn`, that means that `plan`, `execute!`, `execute-one!`, and the "friendly" SQL functions will all accept these options for generating rows and result sets. ## Statements & Prepared Statements diff --git a/doc/getting-started.md b/doc/getting-started.md index 5d39e28..61b65dd 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -93,6 +93,8 @@ Since we used `execute-one!`, we get just one row back (a hash map). This also s If the result set contains no rows, `execute-one!` returns `nil`. When no result is available, and `next.jdbc` returns a fake "result set" containing the "update count", `execute-one!` returns just a single hash map with the key `next.jdbc/update-count` and the number of rows updated. +In the same way that you would use `execute-one!` if you only want one row or one update count, compared to `execute!` for multiple rows or a vector containing an update count, you can also ask `execute!` to return multiple result sets -- such as might be returned from a stored procedure call, or a T-SQL script (for SQL Server) -- instead of just one. If you pass the `:multi-rs true` option to `execute!`, you will get back a vector of results sets, instead of just one result set: a vector of zero or more vectors. The result may well be a mix of vectors containing realized rows and vectors containing update counts, reflecting the results from specific SQL operations in the stored procedure or script. + > Note: In general, you should use `execute-one!` for DDL operations since you will only get back an update count. If you have a SQL statement that you know will only return an update count, `execute-one!` is the right choice. If you have a SQL statement that you know will only return a single row in the result set, you probably want to use `execute-one!`. If you use `execute-one!` for a SQL statement that would return multiple rows in a result set, even though you will only get the first row back (as a hash map), the full result set will still be retrieved from the database -- it does not limit the SQL in any way. ### Options & Result Set Builders diff --git a/src/next/jdbc.clj b/src/next/jdbc.clj index b46283f..9b04e92 100644 --- a/src/next/jdbc.clj +++ b/src/next/jdbc.clj @@ -214,7 +214,10 @@ (defn execute! "General SQL execution function. - Returns a fully-realized result set. + Returns a fully-realized result set. When `:multi-rs true` is provided, will + return multiple result sets, as a vector of result sets. Each result set is + a vector of hash maps, by default, but can be controlled by the `:builder-fn` + option. Can be called on a `PreparedStatement`, a `Connection`, or something that can produce a `Connection` via a `DataSource`."