Hi
pá 11. 9. 2020 v 10:50 odesílatel Pavel Stehule <pavel.steh...@gmail.com>
napsal:
> Hi
>
> po 7. 9. 2020 v 14:14 odesílatel Surafel Temesgen <surafel3...@gmail.com>
> napsal:
>
>> Hi Pavel
>>
>> On Fri, Sep 4, 2020 at 6:22 AM Pavel Stehule <pavel.steh...@gmail.com>
>> wrote:
>>
>>>
>>> Here is updated patch for pg_dump
>>>
>>>
>> pg_dumpall also has –exclude-database=pattern and –no-comments option
>> doesn't that qualify it to benefits from this feature? And please add a
>> test case for this option
>>
>
> This patch is related to pg_dump (in this moment), so pg_dumpall options
> are out of scope.
>
> I am not sure if pg_dumpall needs this functionality - maybe, but I can
> use bash or some similar for implementation of this feature. There is no
> requirement to do it all necessary work under one transaction, one snapshot.
>
> For pg_dump can be used different format, because it uses different
> granularity. Some like "{+/-} dbname"
>
> "--no-comments" is a global parameter without arguments. I don't
> understand how this parameter can be related to this feature?
>
> I am working on regress tests.
>
There is a updated version with regress tests
Regards
Pavel
> Regards
>
> Pavel
>
>
>> regards
>>
>> Surafel
>>
>>
>
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 0b2e2de87b..c4f818f305 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -755,6 +755,56 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Read objects filters from the specified file.
+ If you use "-" as a filename, the filters are read from stdin.
+ The lines of this file must have the following format:
+<synopsis>
+(+|-)[tnfd] <replaceable class="parameter">objectname</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first character specifies whether the object is to be included
+ (<literal>+</literal>) or excluded (<literal>-</literal>), and the
+ second character specifies the type of object to be filtered:
+ <literal>t</literal> (table),
+ <literal>n</literal> (schema),
+ <literal>f</literal> (foreign server),
+ <literal>d</literal> (table data).
+ </para>
+
+ <para>
+ With the following filter file, the dump would include table
+ <literal>mytable1</literal> and data from foreign tables of
+ <literal>some_foreign_server</literal> foreign server, but exclude data
+ from table <literal>mytable2</literal>.
+<programlisting>
++t mytable1
++f some_foreign_server
+-d mytable2
+</programlisting>
+ </para>
+
+ <para>
+ The lines starting with symbol <literal>#</literal> are ignored.
+ Previous white chars (spaces, tabs) are not allowed. These
+ lines can be used for comments, notes. Empty lines are ignored too.
+ </para>
+
+ <para>
+ The <option>--filter</option> option works just like the other
+ options to include or exclude tables, schemas, table data, or foreign
+ tables, and both forms may be combined. Note that there are no options
+ to exclude a specific foreign table or to include a specific table's
+ data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 784bceaec3..94a8b2e187 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -53,9 +53,11 @@
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
#include "common/connect.h"
+#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "getopt_long.h"
+#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
#include "pg_backup_db.h"
@@ -291,6 +293,7 @@ static void appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+static void read_patterns_from_file(char *filename, DumpOptions *dopt);
int
@@ -365,6 +368,7 @@ main(int argc, char **argv)
{"enable-row-security", no_argument, &dopt.enable_row_security, 1},
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
+ {"filter", required_argument, NULL, 12},
{"if-exists", no_argument, &dopt.if_exists, 1},
{"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -604,6 +608,10 @@ main(int argc, char **argv)
optarg);
break;
+ case 12: /* filter implementation */
+ read_patterns_from_file(optarg, &dopt);
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit_nicely(1);
@@ -1023,6 +1031,8 @@ help(const char *progname)
" access to)\n"));
printf(_(" --exclude-table-data=PATTERN do NOT dump data for the specified table(s)\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " from the filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18477,3 +18487,164 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse reloptions array");
}
+
+/*
+ * Print error message and exit.
+ */
+static void
+exit_invalid_filter_format(FILE *fp, char *filename, char *message, char *line, int lineno)
+{
+ pg_log_error("invalid format of filter file \"%s\": %s",
+ filename,
+ message);
+
+ fprintf(stderr, "%d: %s\n", lineno, line);
+
+ if (fp != stdin)
+ fclose(fp);
+
+ exit_nicely(-1);
+}
+
+/*
+ * Read dumped object specification from file
+ */
+static void
+read_patterns_from_file(char *filename, DumpOptions *dopt)
+{
+ FILE *fp;
+ int lineno = 0;
+ StringInfoData line;
+
+ /* use "-" as symbol for stdin */
+ if (strcmp(filename, "-") != 0)
+ {
+ fp = fopen(filename, "r");
+ if (!fp)
+ fatal("could not open the input file \"%s\": %m",
+ filename);
+ }
+ else
+ fp = stdin;
+
+ initStringInfo(&line);
+
+ while (pg_get_line_append(fp, &line))
+ {
+ bool is_include;
+ char objecttype;
+ char *objectname;
+ char *str = line.data;
+
+ lineno += 1;
+
+ (void) pg_strip_crlf(str);
+
+ /* ignore empty rows */
+ if (*str == '\0')
+ goto resetline;
+
+ /* when first char is hash, ignore whole line */
+ if (*str == '#')
+ goto resetline;
+
+ if (str[1] == '\0')
+ exit_invalid_filter_format(fp,
+ filename,
+ "line too short",
+ str,
+ lineno);
+
+ if (str[0] == '+')
+ is_include = true;
+ else if (str[0] == '-')
+ is_include = false;
+ else
+ exit_invalid_filter_format(fp,
+ filename,
+ "invalid option type (use [+-]",
+ str,
+ lineno);
+
+ objecttype = str[1];
+ objectname = &str[2];
+
+ /* skip initial spaces */
+ while (isspace(*objectname))
+ objectname++;
+
+ if (*objectname == '\0')
+ exit_invalid_filter_format(fp,
+ filename,
+ "missing object name",
+ str,
+ lineno);
+
+ if (objecttype == 't')
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns,
+ objectname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns,
+ objectname);
+ }
+ else if (objecttype == 'n')
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objectname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objectname);
+ }
+ else if (objecttype == 'd')
+ {
+ if (is_include)
+ exit_invalid_filter_format(fp,
+ filename,
+ "include filter is not supported for this type of object",
+ str,
+ lineno);
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objectname);
+ }
+ else if (objecttype == 'f')
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objectname);
+ else
+ exit_invalid_filter_format(fp,
+ filename,
+ "exclude filter is not supported for this type of object",
+ str,
+ lineno);
+ }
+ else
+ exit_invalid_filter_format(fp,
+ filename,
+ "invalid object type (use [tndf])",
+ str,
+ lineno);
+
+resetline:
+
+ resetStringInfo(&line);
+ }
+
+ pfree(line.data);
+
+ if (ferror(fp))
+ fatal("could not read from file \"%s\": %m", filename);
+
+ if (fp != stdin)
+ fclose(fp);
+}
diff --git a/src/bin/pg_dump/t/004_pg_dump_filter.pl b/src/bin/pg_dump/t/004_pg_dump_filter.pl
new file mode 100644
index 0000000000..957442c5e4
--- /dev/null
+++ b/src/bin/pg_dump/t/004_pg_dump_filter.pl
@@ -0,0 +1,129 @@
+use strict;
+use warnings;
+
+use Config;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 22;
+
+my $tempdir = TestLib::tempdir;
+my $inputfile;
+
+
+my $node = get_new_node('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres', "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+
+open $inputfile, '>', "$tempdir/inputfile.txt";
+
+print $inputfile "+t table_one\n";
+print $inputfile "+t table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "-d table_one\n";
+close $inputfile;
+
+my ($cmd, $stdout, $stderr, $result);
+command_ok(
+ [ "pg_dump", '-p', $port, "-f", $plainfile, "--filter=$tempdir/inputfile.txt", 'postgres' ],
+ "dump tables with filter");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public.table_three/m, "table three not dumped");
+ok($dump !~ qr/^COPY public.table_one/m, "content of table one is not included");
+ok($dump =~ qr/^COPY public.table_two/m, "content of table two is included");
+
+open $inputfile, '>', "$tempdir/inputfile.txt";
+
+print $inputfile "-t table_one\n";
+close $inputfile;
+
+command_ok(
+ [ "pg_dump", '-p', $port, "-f", $plainfile, "--filter=$tempdir/inputfile.txt", 'postgres' ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public.table_three/m, "dumped table three");
+
+open $inputfile, '>', "$tempdir/inputfile.txt";
+
+print $inputfile "-n public\n";
+close $inputfile;
+
+command_ok(
+ [ "pg_dump", '-p', $port, "-f", $plainfile, "--filter=$tempdir/inputfile.txt", 'postgres' ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+#########################################
+# For test of +f option we need created foreign server or accept
+# fail and check error
+
+open $inputfile, '>', "$tempdir/inputfile.txt";
+
+print $inputfile "+f doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [ "pg_dump", '-p', $port, "-f", $plainfile, "--filter=$tempdir/inputfile.txt", 'postgres' ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump foreign server");
+
+#########################################
+# Test broken input format
+
+open $inputfile, '>', "$tempdir/inputfile.txt";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [ "pg_dump", '-p', $port, "-f", $plainfile, "--filter=$tempdir/inputfile.txt", 'postgres' ],
+ qr/pg_dump: error: invalid format of filter file/,
+ "broken format check");
+
+open $inputfile, '>', "$tempdir/inputfile.txt";
+print $inputfile "+";
+close $inputfile;
+
+command_fails_like(
+ [ "pg_dump", '-p', $port, "-f", $plainfile, "--filter=$tempdir/inputfile.txt", 'postgres' ],
+ qr/pg_dump: error: invalid format of filter file/,
+ "broken format check");
+
+open $inputfile, '>', "$tempdir/inputfile.txt";
+print $inputfile "+d sometable";
+close $inputfile;
+
+command_fails_like(
+ [ "pg_dump", '-p', $port, "-f", $plainfile, "--filter=$tempdir/inputfile.txt", 'postgres' ],
+ qr/include filter is not supported for this type of object/,
+ "broken format check");
+
+open $inputfile, '>', "$tempdir/inputfile.txt";
+print $inputfile "-f someforeignserver";
+close $inputfile;
+
+command_fails_like(
+ [ "pg_dump", '-p', $port, "-f", $plainfile, "--filter=$tempdir/inputfile.txt", 'postgres' ],
+ qr/exclude filter is not supported for this type of object/,
+ "broken format check");
