From aab9db36058dabed1dd93fb7f521e87c1dc98bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Redon?= Date: Tue, 20 Feb 2024 08:37:14 +0100 Subject: [PATCH 1/5] Fix typo in README heading (#81) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73dc171..1bd1d3d 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ When `pg_ivm` is installed, the following objects are created. ### Functions -#### create_imm +#### create_immv Use `create_immv` function to create IMMV. ``` From 8c3b4ba9d7061ac5338ce5ee55e11a66f982d904 Mon Sep 17 00:00:00 2001 From: Yugo Nagata Date: Tue, 27 Feb 2024 17:09:34 +0900 Subject: [PATCH 2/5] Consider tuple duplicity in maintenance of EXISTS views When a tuple is inserted into a table in an EXISTS subquery, the duplicity of row is computed by count(*), but it was not considered and only one tuple was inserted with ignoring the duplicity. This is fixed by duplicating rows as much as the duplicity by using generate_series at inserting. (Issue #82) --- expected/pg_ivm.out | 13 +++++++++++++ matview.c | 22 ++++++++++++++++++++-- sql/pg_ivm.sql | 3 +++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/expected/pg_ivm.out b/expected/pg_ivm.out index bf559d3..4252f62 100644 --- a/expected/pg_ivm.out +++ b/expected/pg_ivm.out @@ -828,6 +828,19 @@ SELECT * FROM mv_ivm_exists_subquery2 ORDER BY i, j; 4 | 40 | 1 (4 rows) +DELETE FROM mv_base_b WHERE i = 1 or i = 3; +INSERT INTO mv_base_b VALUES (1,100), (3,300); +SELECT * FROM mv_ivm_exists_subquery ORDER BY i, j; + i | j | __ivm_exists_count_0__ +---+-----+------------------------ + 1 | 10 | 1 + 1 | 10 | 1 + 3 | 30 | 1 + 3 | 30 | 1 + 3 | 300 | 1 + 4 | 40 | 1 +(6 rows) + ROLLBACK; -- support simple subquery in FROM clause BEGIN; diff --git a/matview.c b/matview.c index d12041e..11a7e13 100644 --- a/matview.c +++ b/matview.c @@ -2523,6 +2523,7 @@ apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, StringInfoData returning_keys; ListCell *lc; char *match_cond = ""; + StringInfoData deltaname_new_for_insert; /* build WHERE condition for searching tuples to be updated */ match_cond = get_matching_condition_string(keys); @@ -2543,6 +2544,24 @@ apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, else appendStringInfo(&returning_keys, "NULL"); + /* + * If count_colname is not "__ivm_count__", the view contains EXISTS + * subquery and the count column to be updated here is "__ivm_exists_count_*" + * that stores the number of columns generated by corresponding EXISTS + * subquery for each row in the view. In this case, __ivm_count__ in + * deltaname_new stores duplicity of rows, and each row need to be + * duplicated as much as __ivm_count__ by using generate_series at + * inserting. + */ + initStringInfo(&deltaname_new_for_insert); + if (!strcmp(count_colname, "__ivm_count__")) + appendStringInfo(&deltaname_new_for_insert, "%s", deltaname_new); + else + appendStringInfo(&deltaname_new_for_insert, + "(SELECT diff.* FROM %s diff," + "pg_catalog.generate_series(1, diff.\"__ivm_count__\"))", + deltaname_new); + /* Search for matching tuples from the view and update if found or insert if not. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2561,9 +2580,8 @@ apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, match_cond, returning_keys.data, matviewname, target_list->data, - target_list->data, deltaname_new, + target_list->data, deltaname_new_for_insert.data, match_cond); - if (SPI_exec(querybuf.data, 0) != SPI_OK_INSERT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); } diff --git a/sql/pg_ivm.sql b/sql/pg_ivm.sql index f99224c..8294e07 100644 --- a/sql/pg_ivm.sql +++ b/sql/pg_ivm.sql @@ -269,6 +269,9 @@ DELETE FROM mv_base_a WHERE (i,j) = (1,60); DELETE FROM mv_base_b WHERE i = 2; SELECT * FROM mv_ivm_exists_subquery ORDER BY i, j; SELECT * FROM mv_ivm_exists_subquery2 ORDER BY i, j; +DELETE FROM mv_base_b WHERE i = 1 or i = 3; +INSERT INTO mv_base_b VALUES (1,100), (3,300); +SELECT * FROM mv_ivm_exists_subquery ORDER BY i, j; ROLLBACK; -- support simple subquery in FROM clause From 01f0ea0eb1d72f2b8bd5430086bce4479f7fcc56 Mon Sep 17 00:00:00 2001 From: Yugo Nagata Date: Tue, 27 Feb 2024 19:32:45 +0900 Subject: [PATCH 3/5] Fix for view using both DISTINCT and EXISTS In previous commit, maintenance of views using EXISTS and containing duplicated tuples was fixed, but it was insufficient because it raised an error during maintenance of views using both DISTINCT and EXISTS. The cause was that tuples were tried to be duplicated even when DISTINCT was specified. In this commit, it is fixed not to duplicate tuples when DISTINCT is used. --- expected/pg_ivm.out | 17 +++++++++++++++++ matview.c | 12 +++++++----- sql/pg_ivm.sql | 4 +++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/expected/pg_ivm.out b/expected/pg_ivm.out index 4252f62..a62dc63 100644 --- a/expected/pg_ivm.out +++ b/expected/pg_ivm.out @@ -828,6 +828,14 @@ SELECT * FROM mv_ivm_exists_subquery2 ORDER BY i, j; 4 | 40 | 1 (4 rows) +--- EXISTS subquery with tuple duplication and DISTINCT +SELECT create_immv('mv_ivm_exists_subquery_distinct', 'SELECT DISTINCT a.i, a.j FROM mv_base_a a WHERE EXISTS(SELECT 1 FROM mv_base_b b WHERE a.i = b.i)'); +NOTICE: created index "mv_ivm_exists_subquery_distinct_index" on immv "mv_ivm_exists_subquery_distinct" + create_immv +------------- + 4 +(1 row) + DELETE FROM mv_base_b WHERE i = 1 or i = 3; INSERT INTO mv_base_b VALUES (1,100), (3,300); SELECT * FROM mv_ivm_exists_subquery ORDER BY i, j; @@ -841,6 +849,15 @@ SELECT * FROM mv_ivm_exists_subquery ORDER BY i, j; 4 | 40 | 1 (6 rows) +SELECT * FROM mv_ivm_exists_subquery_distinct ORDER BY i, j; + i | j | __ivm_exists_count_0__ | __ivm_count__ +---+-----+------------------------+--------------- + 1 | 10 | 1 | 2 + 3 | 30 | 1 | 2 + 3 | 300 | 1 | 1 + 4 | 40 | 1 | 1 +(4 rows) + ROLLBACK; -- support simple subquery in FROM clause BEGIN; diff --git a/matview.c b/matview.c index 11a7e13..6f8525b 100644 --- a/matview.c +++ b/matview.c @@ -195,7 +195,7 @@ static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, - const char* count_colname); + const char* count_colname, bool distinct); static char *get_matching_condition_string(List *keys); static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); @@ -2033,7 +2033,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply new delta */ if (use_count) apply_new_delta_with_count(matviewname, NEW_DELTA_ENRNAME, - keys, aggs_set_new, &target_list_buf, count_colname); + keys, aggs_set_new, &target_list_buf, count_colname, + query->distinctClause != NULL); else apply_new_delta(matviewname, NEW_DELTA_ENRNAME, &target_list_buf); } @@ -2517,7 +2518,7 @@ apply_old_delta(const char *matviewname, const char *deltaname_old, static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo aggs_set, StringInfo target_list, - const char* count_colname) + const char* count_colname, bool distinct) { StringInfoData querybuf; StringInfoData returning_keys; @@ -2525,6 +2526,7 @@ apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, char *match_cond = ""; StringInfoData deltaname_new_for_insert; + /* build WHERE condition for searching tuples to be updated */ match_cond = get_matching_condition_string(keys); @@ -2551,10 +2553,10 @@ apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, * subquery for each row in the view. In this case, __ivm_count__ in * deltaname_new stores duplicity of rows, and each row need to be * duplicated as much as __ivm_count__ by using generate_series at - * inserting. + * inserting if DISTINCT is not used. */ initStringInfo(&deltaname_new_for_insert); - if (!strcmp(count_colname, "__ivm_count__")) + if (!strcmp(count_colname, "__ivm_count__") || distinct) appendStringInfo(&deltaname_new_for_insert, "%s", deltaname_new); else appendStringInfo(&deltaname_new_for_insert, diff --git a/sql/pg_ivm.sql b/sql/pg_ivm.sql index 8294e07..87adf90 100644 --- a/sql/pg_ivm.sql +++ b/sql/pg_ivm.sql @@ -269,9 +269,12 @@ DELETE FROM mv_base_a WHERE (i,j) = (1,60); DELETE FROM mv_base_b WHERE i = 2; SELECT * FROM mv_ivm_exists_subquery ORDER BY i, j; SELECT * FROM mv_ivm_exists_subquery2 ORDER BY i, j; +--- EXISTS subquery with tuple duplication and DISTINCT +SELECT create_immv('mv_ivm_exists_subquery_distinct', 'SELECT DISTINCT a.i, a.j FROM mv_base_a a WHERE EXISTS(SELECT 1 FROM mv_base_b b WHERE a.i = b.i)'); DELETE FROM mv_base_b WHERE i = 1 or i = 3; INSERT INTO mv_base_b VALUES (1,100), (3,300); SELECT * FROM mv_ivm_exists_subquery ORDER BY i, j; +SELECT * FROM mv_ivm_exists_subquery_distinct ORDER BY i, j; ROLLBACK; -- support simple subquery in FROM clause @@ -295,7 +298,6 @@ SELECT create_immv('mv_ivm_subquery', 'SELECT a.j FROM mv_base_a a WHERE EXISTS( SELECT create_immv('mv_ivm_subquery', 'SELECT EXISTS(SELECT 1 from mv_base_b) FROM mv_base_a a'); SELECT create_immv('mv_ivm_subquery', 'SELECT false OR EXISTS(SELECT 1 FROM mv_base_a) FROM mv_base_b'); SELECT create_immv('mv_ivm_subquery', 'SELECT * FROM generate_series(1, CASE EXISTS(SELECT 1 FROM mv_base_a) WHEN true THEN 100 ELSE 10 END), mv_base_b'); - -- support join subquery in FROM clause BEGIN; SELECT create_immv('mv_ivm_join_subquery', 'SELECT i, j, k FROM ( SELECT i, a.j, b.k FROM mv_base_b b INNER JOIN mv_base_a a USING(i)) tmp'); From 8f5bb5300a725931d98b4ade6a25f7e02b093f41 Mon Sep 17 00:00:00 2001 From: Colin Zhao Date: Fri, 1 Mar 2024 13:13:38 +0800 Subject: [PATCH 4/5] Check if PgIvmImmvRelationId is invalid before open it (#78) When pg_ivm is installed shared_preload_libraries without executing CREATE EXTENSION command, the hook function is set while the catalog table pg_ivm_immv is not created. In this case, the hook function failed to open the catalog table and an error was raised. Although this way of installing pg_ivm was not considered, it would be nice to reduce any possible troubles on users. Therefore, to prevent this error, check if PgIvmImmvRelationId is invalid before open it. When this is invalid, pg_ivm_immv relation is not created yet, so there are not any IMMVs, so we don't have to do anything in this hook function. Review and commit message by Yugo Nagata --- pg_ivm.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pg_ivm.c b/pg_ivm.c index 33b99c6..ba14888 100644 --- a/pg_ivm.c +++ b/pg_ivm.c @@ -385,11 +385,16 @@ PgIvmObjectAccessHook(ObjectAccessType access, Oid classId, if (access == OAT_DROP && classId == RelationRelationId && !OidIsValid(subId)) { - Relation pgIvmImmv = table_open(PgIvmImmvRelationId(), AccessShareLock); + Relation pgIvmImmv; SysScanDesc scan; ScanKeyData key; HeapTuple tup; - + Oid pgIvmImmvOid = PgIvmImmvRelationId(); + + if (pgIvmImmvOid == InvalidOid) + return; + + pgIvmImmv = table_open(pgIvmImmvOid, AccessShareLock); ScanKeyInit(&key, Anum_pg_ivm_immv_immvrelid, BTEqualStrategyNumber, F_OIDEQ, From d67995c0ab4c48c92b16fece5758ca14134615a3 Mon Sep 17 00:00:00 2001 From: Yugo Nagata Date: Fri, 1 Mar 2024 14:48:55 +0900 Subject: [PATCH 5/5] Fix an error raised when dropping pg_ivm extension Previously, DROP EXTENSION pg_ivm failed due to the failure of opening the index on pg_ivm_immv in PgIvmObjectAccessHook that is called on dropping pg_ivm_immv, because when pg_ivm_immv is being dropped, the index on it is already dropped. This is fixed to return immediately from the hook function if the dropped table is pg_ivm_immv. --- pg_ivm.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pg_ivm.c b/pg_ivm.c index ba14888..a1e6f27 100644 --- a/pg_ivm.c +++ b/pg_ivm.c @@ -390,9 +390,18 @@ PgIvmObjectAccessHook(ObjectAccessType access, Oid classId, ScanKeyData key; HeapTuple tup; Oid pgIvmImmvOid = PgIvmImmvRelationId(); - + + /* pg_ivm_immv is not created yet, so there are no IMMVs, either. */ if (pgIvmImmvOid == InvalidOid) return; + + /* + * When the dropped table is pg_ivm_immv, we don't need to continue + * any more. Also, in this case, the index on it is already dropped, + * so the index scan below will fail and raise an error. + */ + if (objectId == pgIvmImmOid) + return; pgIvmImmv = table_open(pgIvmImmvOid, AccessShareLock); ScanKeyInit(&key,