From 48359d69107da581d004da37230f949cb8796eb8 Mon Sep 17 00:00:00 2001 From: Peter Smith Date: Fri, 4 Mar 2022 14:32:33 +1100 Subject: [PATCH v3] Update the documentation for logical replication. This patch introduces a new documentation page for describing the "Row Filtering" feature. Author: Peter Smith Reviewed By: Greg Nancarrow, Aleksander Alekseev, Amit Kapila, Ajin Cherian Discussion: https://www.postgresql.org/message-id/CAHut%2BPtnsBr59%3D_NvxXp_%3DS-em0WxyuDOQmSTuHGb4sVhkHffg%40mail.gmail.com --- doc/src/sgml/logical-replication.sgml | 521 +++++++++++++++++++++++++++++++ doc/src/sgml/ref/create_publication.sgml | 2 + 2 files changed, 523 insertions(+) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 96b4886..657b460 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -118,6 +118,8 @@ any combination of INSERT, UPDATE, DELETE, and TRUNCATE, similar to how triggers are fired by particular event types. By default, all operation types are replicated. + (Row filters have no effect for TRUNCATE. See + ). @@ -317,6 +319,525 @@ + + Row Filters + + + By default, all data from all published tables will be replicated to the + appropriate subscribers. + + + + The default data replication can be reduced by using row filters. + A user might choose to use row filters for behavioral or for performance + reasons. + + + + With row filtering, the decision whether to replicate row data depends on + the row data values. Rows that don't satisfy an optional WHERE + clause will be filtered out. This allows a database or set of tables to be + partially replicated. + + + + Use the PUBLICATION ... FOR TABLE + + table_name WHERE ( + expression) + to specify which rows will be replicated. + + + + The row filtering is defined per table. A new row filter can be added simply + by specifying a WHERE clause after the table name. The + WHERE clause must be enclosed by parentheses. + + + + Row filter Rules + + + If the row filter evaluates to false or + NULL then the row is not replicated. + + + + Row filters are applied before publishing the changes. + + + + The WHERE clause expression is evaluated with the same + role used for the replication connection. + + + + Row filters have no effect for TRUNCATE commands. + + + + + + UPDATE transformations + + + Whenever an UPDATE is processed, the row filter + expression is evaluated for both the old and new row (i.e. before + and after the data is updated). + + + + If both evaluations are true, it replicates the + UPDATE. + + + + If both evaluations are false, it doesn't replicate + anything. + + + + If only one of the old/new rows matches the row filter expression, the + UPDATE is transformed to INSERT or + DELETE, to avoid any data inconsistency, as follows: + +Case 1: old-row (no match) new-row (no match) --> (drop change) +Case 2: old-row (no match) new row (match) --> INSERT +Case 3: old-row (match) new-row (no match) --> DELETE +Case 4: old-row (match) new row (match) --> UPDATE + + + + + + + Partition tables + + + If the publication contains a partitioned table, the publication parameter + publish_via_partition_root determines which row filter + is used. + + + + + If publish_via_partition_root is false + (default), each partition's row filter is used. + + + + + + If publish_via_partition_root is true, + the root partitioned table's row filter is used. + + + + + + + + + + WHERE clause Expression Restrictions + + + The row filter WHERE clause for a table added to a + publication that publishes UPDATE and/or + DELETE operations must contain only columns that are + covered by the replica identity. The row filter + WHERE clause for a table added to a publication that + publishes INSERT can use any column. + + + + The WHERE clause allows only simple expressions that + don't have user-defined functions, operators, non-immutable built-in + functions, or references to system columns. + + + + + + Subscriber initial table synchronization + + + If you choose to do the initial table synchronization, only data that + satisfies the row filters is copied to the subscriber. + + + + If the subscription has several publications in which a table has been + published with different WHERE clauses, rows that satisfy + any of the expressions will be copied. (see + ). + + + + + Publication operations of filters are ignored during initial table + synchronization. + + + + + + If a subscriber is a pre-15 version, the initial table synchronization + won't use row filters even if they are defined in the publication. + + + + + + + Combining multiple row filters for the same table + + + If the subscription has several publications in which the same table has + been published with different row filters (for the same publish + operation), those expressions get OR'ed together, so that rows satisfying + any of the expressions will be replicated. + + + + This means all the other filters (for the same table) become redundant if: + + + + + one of the publications has no filter at all. + + + + + + one of the publications was created using FOR ALL TABLES. + + + + + + one of the publications was created using + FOR ALL TABLES IN SCHEMA and the table belongs to + that schema. + + + + + + + + + + PSQL commands to display Row Filters + + + PSQL commands \dRp+ and \d will + display row filter information. + + + + + + Examples + + + Create some tables to be used in the following examples. + +test_pub=# CREATE TABLE t1(a int, b int, c text, primary key(a,c)); +CREATE TABLE +test_pub=# CREATE TABLE t2(d int primary key, e int, f int); +CREATE TABLE +test_pub=# CREATE TABLE t3(g int primary key, h int, i int); +CREATE TABLE + + + + + Create some publications. + + + + Notice that publication p2 has tables with and without + row filters. + + + + Notice that publication p3 has row filters on multiple + tables. + +test_pub=# CREATE PUBLICATION p1 FOR TABLE t1 WHERE (a > 5 AND c = 'NSW'); +CREATE PUBLICATION +test_pub=# CREATE PUBLICATION p2 FOR TABLE t1, t2 WHERE (e = 99); +CREATE PUBLICATION +test_pub=# CREATE PUBLICATION p3 FOR TABLE t2 WHERE (d = 10), t3 WHERE (g = 10); +CREATE PUBLICATION + + + + + The PSQL command \dRp+ shows what row filters are + defined for the publications. + +test_pub=# \dRp+ + Publication p1 + Owner | All tables | Inserts | Updates | Deletes | Truncates | Via root +----------+------------+---------+---------+---------+-----------+---------- + postgres | f | t | t | t | t | f +Tables: + "public.t1" WHERE ((a > 5) AND (c = 'NSW'::text)) + + Publication p2 + Owner | All tables | Inserts | Updates | Deletes | Truncates | Via root +----------+------------+---------+---------+---------+-----------+---------- + postgres | f | t | t | t | t | f +Tables: + "public.t1" + "public.t2" WHERE (e = 99) + + Publication p3 + Owner | All tables | Inserts | Updates | Deletes | Truncates | Via root +----------+------------+---------+---------+---------+-----------+---------- + postgres | f | t | t | t | t | f +Tables: + "public.t2" WHERE (d = 10) + "public.t3" WHERE (g = 10) + + + + + The PSQL command \d shows what publications the table is + a member of, as well as that table's row filter in those publications. + + + + Notice that table t1 is a member of two publications, but + has a WHERE clause only for p1. + + + + Notice that table t2 is a member of two publications, and + has a different WHERE clause for each of them. + +test_pub=# \d t1 + Table "public.t1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | not null | + b | integer | | | + c | text | | not null | +Indexes: + "t1_pkey" PRIMARY KEY, btree (a, c) +Publications: + "p1" WHERE ((a > 5) AND (c = 'NSW'::text)) + "p2" + + +test_pub=# \d t2 + Table "public.t2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + d | integer | | not null | + e | integer | | | + f | integer | | | +Indexes: + "t2_pkey" PRIMARY KEY, btree (d) +Publications: + "p2" WHERE (e = 99) + "p3" WHERE (d = 10) + + +test_pub=# \d t3 + Table "public.t3" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + g | integer | | not null | + h | integer | | | + i | integer | | | +Indexes: + "t3_pkey" PRIMARY KEY, btree (g) +Publications: + "p3" WHERE (g = 10) + + + + + On the subscriber node, create a table t1 and a + subscription s1 to publication p1. + +test_sub=# CREATE TABLE t1(a int, b int, c text, primary key(a,c)); +CREATE TABLE +test_sub=# CREATE SUBSCRIPTION s1 +test_sub-# CONNECTION 'host=localhost dbname=test_pub application_name=s1' +test_sub-# PUBLICATION p1; +CREATE SUBSCRIPTION + + + + + INSERT some data. + + + + Notice that only the rows satisfying the t1 WHERE + clause of publication p1 are replicated. + +test_pub=# INSERT INTO t1 VALUES (2, 102, 'NSW'); +INSERT 0 1 +test_pub=# INSERT INTO t1 VALUES (3, 103, 'QLD'); +INSERT 0 1 +test_pub=# INSERT INTO t1 VALUES (4, 104, 'VIC'); +INSERT 0 1 +test_pub=# INSERT INTO t1 VALUES (5, 105, 'ACT'); +INSERT 0 1 +test_pub=# INSERT INTO t1 VALUES (6, 106, 'NSW'); +INSERT 0 1 +test_pub=# INSERT INTO t1 VALUES (7, 107, 'NT'); +INSERT 0 1 +test_pub=# INSERT INTO t1 VALUES (8, 108, 'QLD'); +INSERT 0 1 +test_pub=# INSERT INTO t1 VALUES (9, 109, 'NSW'); +INSERT 0 1 + + +test_pub=# SELECT * FROM t1; + a | b | c +---+-----+----- + 2 | 102 | NSW + 3 | 103 | QLD + 4 | 104 | VIC + 5 | 105 | ACT + 6 | 106 | NSW + 7 | 107 | NT + 8 | 108 | QLD + 9 | 109 | NSW +(8 rows) + + +test_sub=# SELECT * FROM t1; + a | b | c +---+-----+----- + 6 | 106 | NSW + 9 | 109 | NSW +(2 rows) + + + + + UPDATE some data, where the old and new values both + satisfy the t1 WHERE clause of publication + p1. + + + + Notice this replicates as a normal UPDATE. + +test_pub=# UPDATE t1 SET b = 999 WHERE a = 6; +UPDATE 1 + + +test_pub=# SELECT * FROM t1; + a | b | c +---+-----+----- + 2 | 102 | NSW + 3 | 103 | QLD + 4 | 104 | VIC + 5 | 105 | ACT + 7 | 107 | NT + 8 | 108 | QLD + 9 | 109 | NSW + 6 | 999 | NSW +(8 rows) + + +test_sub=# SELECT * FROM t1; + a | b | c +---+-----+----- + 9 | 109 | NSW + 6 | 999 | NSW +(2 rows) + + + + + UPDATE some data, where the old values did not satisfy + the t1 WHERE clause of publication p1, + but the new values do satisfy it. + + + + Notice the UPDATE is transformed and replicates as an + INSERT. + +test_pub=# UPDATE t1 SET a = 999 WHERE a = 2; +UPDATE 1 + + +test_pub=# SELECT * FROM t1; + a | b | c +-----+-----+----- + 3 | 103 | QLD + 4 | 104 | VIC + 5 | 105 | ACT + 7 | 107 | NT + 8 | 108 | QLD + 9 | 109 | NSW + 6 | 999 | NSW + 999 | 102 | NSW +(8 rows) + + +test_sub=# SELECT * FROM t1; + a | b | c +-----+-----+----- + 9 | 109 | NSW + 6 | 999 | NSW + 999 | 102 | NSW +(3 rows) + + + + + UPDATE some data, where the old values satisfied + the t1 WHERE clause of publication p1, + but the new values do not satisfy it. + + + + Notice the UPDATE is transformed and replicates as a + DELETE. + +test_pub=# UPDATE t1 SET c = 'VIC' WHERE a = 9; +UPDATE 1 + + +test_pub=# SELECT * FROM t1; + a | b | c +-----+-----+----- + 3 | 103 | QLD + 4 | 104 | VIC + 5 | 105 | ACT + 7 | 107 | NT + 8 | 108 | QLD + 6 | 999 | NSW + 999 | 102 | NSW + 9 | 109 | VIC +(8 rows) + + +test_sub=# SELECT * FROM t1; + a | b | c +-----+-----+----- + 6 | 999 | NSW + 999 | 102 | NSW +(2 rows) + + + + + + + Conflicts diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 4979b9b..cb16e97 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -247,6 +247,8 @@ CREATE PUBLICATION name publish_via_partition_root determines if it uses the partition's row filter (if the parameter is false, the default) or the root partitioned table's row filter. + See for more details about + row filters. -- 1.8.3.1