On Sat, Nov 28, 2020 at 09:14:35PM +0100, Pavel Stehule wrote: > Any short or long option can be read from this file in simple format - one > option per line. Arguments inside double quotes can be multi lined. Row > comments started by # and can be used everywhere.
Does this support even funkier table names ? This tests a large number and fraction of characters in dbname/username, so all of pg_dump has to continue supporting that: ./src/bin/pg_dump/t/010_dump_connstr.pl I tested and it seems to work with -t "fooƄ" But it didn't work with -t "foo\nbar" (literal newline). Fix attached. If you send another patch, please consider including a test case for quoted names in long and short options. > +static char *optsfilename = NULL; > + * It assign the values of options to related DumpOption fields or to > + * some global values. It is called from twice. First, for processing > + * the command line argumens. Second, for processing an options from > + * options file. This didn't support multiple config files, nor config files which include config files, as Dean and I mentioned. I think the argument parsers should themselves call the config file parser, as need be, so the last option specification should override previous ones. For example pg_dump --config-file=./pg_dump.conf --blobs should have blobs even if the config file says --no-blobs. (Command-line arguments normally take precedence over config files, certainly if the argument is specified "later"). I think it'd be ok if it's recursive. I made a quick hack to do that. I doubt this will satisfy Stephen. Personally, I would use this if it were a plain and simple text config file (which for our purposes I would pass on stdin), and I would almost certainly not use it if it were json. But it'd be swell if there were a standard config file format, that handled postgresql.conf and maybe pg_hba.conf. -- Justin
>From d9bac559d235f568a3419070205f05d77853a87f Mon Sep 17 00:00:00 2001 From: Justin Pryzby <pryz...@telsasoft.com> Date: Sat, 28 Nov 2020 15:53:11 -0600 Subject: [PATCH 1/4] Re: proposal: possibility to read dumped table's name from file Pavel Stehule Nov 28 --- doc/src/sgml/ref/pg_dump.sgml | 28 + src/bin/pg_dump/pg_dump.c | 817 ++++++++++++++++------ src/bin/pg_dump/t/004_pg_dump_optsfile.pl | 165 +++++ 3 files changed, 812 insertions(+), 198 deletions(-) create mode 100644 src/bin/pg_dump/t/004_pg_dump_optsfile.pl diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 0aa35cf0c3..efdb53f069 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -956,6 +956,34 @@ PostgreSQL documentation </listitem> </varlistentry> + <varlistentry> + <term><option>--options-file=<replaceable class="parameter">filename</replaceable></option></term> + <listitem> + <para> + Read options from file (one option per line). Short or long options + are supported. If you use "-" as a filename, the filters are read + from stdin. + </para> + + <para> + With the following options file, the dump would include table + <literal>mytable1</literal> and data from foreign tables of + <literal>some_foreign_server</literal> foreign server, but exclude data + from table <literal>mytable2</literal>. +<programlisting> +-t mytable1 +--include-foreign-data=some_foreign_server +--exclude-table-data=mytable2 +</programlisting> + </para> + + <para> + The text after symbol <literal>#</literal> is ignored. This can + be used for comments, notes. Empty lines are ignored too. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><option>--quote-all-identifiers</option></term> <listitem> diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index dc1d41dd8d..96acb8a1d5 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -54,9 +54,11 @@ #include "catalog/pg_trigger_d.h" #include "catalog/pg_type_d.h" #include "common/connect.h" +#include "common/string.h" #include "dumputils.h" #include "fe_utils/string_utils.h" #include "getopt_long.h" +#include "lib/stringinfo.h" #include "libpq/libpq-fs.h" #include "parallel.h" #include "pg_backup_db.h" @@ -129,6 +131,17 @@ static const CatalogId nilCatalogId = {0, 0}; static bool have_extra_float_digits = false; static int extra_float_digits; +static const char *filename = NULL; +static const char *format = "p"; +static bool g_verbose = false; +static const char *dumpencoding = NULL; +static const char *dumpsnapshot = NULL; +static char *use_role = NULL; +static long rowsPerInsert; +static int numWorkers = 1; +static int compressLevel = -1; +static char *optsfilename = NULL; + /* * The default number of rows per INSERT when * --inserts is specified without --rows-per-insert @@ -294,14 +307,226 @@ static void appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions, static char *get_synchronized_snapshot(Archive *fout); static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +static void read_options_from_file(char *filename, + DumpOptions *dopt, + const char *optstring, + const struct option *longopts, + const char *progname); + +#define OPTIONS_FILE_OPT_NUMBER 12 + +/* + * It assign the values of options to related DumpOption fields or to + * some global values. It is called from twice. First, for processing + * the command line argumens. Second, for processing an options from + * options file. + */ +static bool +process_option(int opt, + char *optargstr, + DumpOptions *dopt, + const char *progname) +{ + char *endptr; + + switch (opt) + { + case 'a': /* Dump data only */ + dopt->dataOnly = true; + break; + + case 'b': /* Dump blobs */ + dopt->outputBlobs = true; + break; + + case 'B': /* Don't dump blobs */ + dopt->dontOutputBlobs = true; + break; + + case 'c': /* clean (i.e., drop) schema prior to create */ + dopt->outputClean = 1; + break; + + case 'C': /* Create DB */ + dopt->outputCreateDB = 1; + break; + + case 'd': /* database name */ + dopt->cparams.dbname = pg_strdup(optargstr); + break; + + case 'E': /* Dump encoding */ + dumpencoding = pg_strdup(optargstr); + break; + + case 'f': + filename = pg_strdup(optargstr); + break; + + case 'F': + format = pg_strdup(optargstr); + break; + + case 'h': /* server host */ + dopt->cparams.pghost = pg_strdup(optargstr); + break; + + case 'j': /* number of dump jobs */ + numWorkers = atoi(optargstr); + break; + + case 'n': /* include schema(s) */ + simple_string_list_append(&schema_include_patterns, optargstr); + dopt->include_everything = false; + break; + + case 'N': /* exclude schema(s) */ + simple_string_list_append(&schema_exclude_patterns, optargstr); + break; + + case 'O': /* Don't reconnect to match owner */ + dopt->outputNoOwner = 1; + break; + + case 'p': /* server port */ + dopt->cparams.pgport = pg_strdup(optargstr); + break; + + case 'R': + /* no-op, still accepted for backwards compatibility */ + break; + + case 's': /* dump schema only */ + dopt->schemaOnly = true; + break; + + case 'S': /* Username for superuser in plain text output */ + dopt->outputSuperuser = pg_strdup(optargstr); + break; + + case 't': /* include table(s) */ + simple_string_list_append(&table_include_patterns, optargstr); + dopt->include_everything = false; + break; + + case 'T': /* exclude table(s) */ + simple_string_list_append(&table_exclude_patterns, optargstr); + break; + + case 'U': + dopt->cparams.username = pg_strdup(optargstr); + break; + + case 'v': /* verbose */ + g_verbose = true; + pg_logging_increase_verbosity(); + break; + + case 'w': + dopt->cparams.promptPassword = TRI_NO; + break; + + case 'W': + dopt->cparams.promptPassword = TRI_YES; + break; + + case 'x': /* skip ACL dump */ + dopt->aclsSkip = true; + break; + + case 'Z': /* Compression Level */ + compressLevel = atoi(optargstr); + if (compressLevel < 0 || compressLevel > 9) + { + pg_log_error("compression level must be in range 0..9"); + return false; + } + break; + + case 0: + /* This covers the long options. */ + break; + + case 2: /* lock-wait-timeout */ + dopt->lockWaitTimeout = pg_strdup(optargstr); + break; + + case 3: /* SET ROLE */ + use_role = pg_strdup(optargstr); + break; + + case 4: /* exclude table(s) data */ + simple_string_list_append(&tabledata_exclude_patterns, optargstr); + break; + + case 5: /* section */ + set_dump_section(optargstr, &dopt->dumpSections); + break; + + case 6: /* snapshot */ + dumpsnapshot = pg_strdup(optargstr); + break; + + case 7: /* no-sync */ + dosync = false; + break; + + case 8: + have_extra_float_digits = true; + extra_float_digits = atoi(optargstr); + if (extra_float_digits < -15 || extra_float_digits > 3) + { + pg_log_error("extra_float_digits must be in range -15..3"); + return false; + } + break; + + case 9: /* inserts */ + + /* + * dump_inserts also stores --rows-per-insert, careful not to + * overwrite that. + */ + if (dopt->dump_inserts == 0) + dopt->dump_inserts = DUMP_DEFAULT_ROWS_PER_INSERT; + break; + case 10: /* rows per insert */ + errno = 0; + rowsPerInsert = strtol(optargstr, &endptr, 10); + + if (endptr == optargstr || *endptr != '\0' || + rowsPerInsert <= 0 || rowsPerInsert > INT_MAX || + errno == ERANGE) + { + pg_log_error("rows-per-insert must be in range %d..%d", + 1, INT_MAX); + return false; + } + dopt->dump_inserts = (int) rowsPerInsert; + break; + + case 11: /* include foreign data */ + simple_string_list_append(&foreign_servers_include_patterns, + optargstr); + break; + + case OPTIONS_FILE_OPT_NUMBER: /* filter implementation */ + optsfilename = pg_strdup(optargstr); + break; + + default: + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + return false; + } + + return true; +} int main(int argc, char **argv) { int c; - const char *filename = NULL; - const char *format = "p"; TableInfo *tblinfo; int numTables; DumpableObject **dobjs; @@ -309,20 +534,11 @@ main(int argc, char **argv) DumpableObject *boundaryObjs; int i; int optindex; - char *endptr; RestoreOptions *ropt; Archive *fout; /* the script file */ - bool g_verbose = false; - const char *dumpencoding = NULL; - const char *dumpsnapshot = NULL; - char *use_role = NULL; - long rowsPerInsert; - int numWorkers = 1; - int compressLevel = -1; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; - static DumpOptions dopt; static struct option long_options[] = { @@ -387,13 +603,17 @@ main(int argc, char **argv) {"no-subscriptions", no_argument, &dopt.no_subscriptions, 1}, {"no-sync", no_argument, NULL, 7}, {"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1}, + {"options-file", required_argument, NULL, OPTIONS_FILE_OPT_NUMBER}, {"rows-per-insert", required_argument, NULL, 10}, {"include-foreign-data", required_argument, NULL, 11}, {"index-collation-versions-unknown", no_argument, &dopt.coll_unknown, 1}, + {"include-foreign-data-file", required_argument, NULL, 17}, {NULL, 0, NULL, 0} }; + const char *short_options = "abBcCd:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:"; + pg_logging_init(argv[0]); pg_logging_set_level(PG_LOG_WARNING); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_dump")); @@ -422,197 +642,20 @@ main(int argc, char **argv) InitDumpOptions(&dopt); - while ((c = getopt_long(argc, argv, "abBcCd:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:", + while ((c = getopt_long(argc, argv, short_options, long_options, &optindex)) != -1) { - switch (c) - { - case 'a': /* Dump data only */ - dopt.dataOnly = true; - break; - - case 'b': /* Dump blobs */ - dopt.outputBlobs = true; - break; - - case 'B': /* Don't dump blobs */ - dopt.dontOutputBlobs = true; - break; - - case 'c': /* clean (i.e., drop) schema prior to create */ - dopt.outputClean = 1; - break; - - case 'C': /* Create DB */ - dopt.outputCreateDB = 1; - break; - - case 'd': /* database name */ - dopt.cparams.dbname = pg_strdup(optarg); - break; - - case 'E': /* Dump encoding */ - dumpencoding = pg_strdup(optarg); - break; - - case 'f': - filename = pg_strdup(optarg); - break; - - case 'F': - format = pg_strdup(optarg); - break; - - case 'h': /* server host */ - dopt.cparams.pghost = pg_strdup(optarg); - break; - - case 'j': /* number of dump jobs */ - numWorkers = atoi(optarg); - break; - - case 'n': /* include schema(s) */ - simple_string_list_append(&schema_include_patterns, optarg); - dopt.include_everything = false; - break; - - case 'N': /* exclude schema(s) */ - simple_string_list_append(&schema_exclude_patterns, optarg); - break; - - case 'O': /* Don't reconnect to match owner */ - dopt.outputNoOwner = 1; - break; - - case 'p': /* server port */ - dopt.cparams.pgport = pg_strdup(optarg); - break; - - case 'R': - /* no-op, still accepted for backwards compatibility */ - break; - - case 's': /* dump schema only */ - dopt.schemaOnly = true; - break; - - case 'S': /* Username for superuser in plain text output */ - dopt.outputSuperuser = pg_strdup(optarg); - break; - - case 't': /* include table(s) */ - simple_string_list_append(&table_include_patterns, optarg); - dopt.include_everything = false; - break; - - case 'T': /* exclude table(s) */ - simple_string_list_append(&table_exclude_patterns, optarg); - break; - - case 'U': - dopt.cparams.username = pg_strdup(optarg); - break; - - case 'v': /* verbose */ - g_verbose = true; - pg_logging_increase_verbosity(); - break; - - case 'w': - dopt.cparams.promptPassword = TRI_NO; - break; - - case 'W': - dopt.cparams.promptPassword = TRI_YES; - break; - - case 'x': /* skip ACL dump */ - dopt.aclsSkip = true; - break; - - case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } - break; - - case 0: - /* This covers the long options. */ - break; - - case 2: /* lock-wait-timeout */ - dopt.lockWaitTimeout = pg_strdup(optarg); - break; - - case 3: /* SET ROLE */ - use_role = pg_strdup(optarg); - break; - - case 4: /* exclude table(s) data */ - simple_string_list_append(&tabledata_exclude_patterns, optarg); - break; - - case 5: /* section */ - set_dump_section(optarg, &dopt.dumpSections); - break; - - case 6: /* snapshot */ - dumpsnapshot = pg_strdup(optarg); - break; - - case 7: /* no-sync */ - dosync = false; - break; - - case 8: - have_extra_float_digits = true; - extra_float_digits = atoi(optarg); - if (extra_float_digits < -15 || extra_float_digits > 3) - { - pg_log_error("extra_float_digits must be in range -15..3"); - exit_nicely(1); - } - break; - - case 9: /* inserts */ - - /* - * dump_inserts also stores --rows-per-insert, careful not to - * overwrite that. - */ - if (dopt.dump_inserts == 0) - dopt.dump_inserts = DUMP_DEFAULT_ROWS_PER_INSERT; - break; - - case 10: /* rows per insert */ - errno = 0; - rowsPerInsert = strtol(optarg, &endptr, 10); - - if (endptr == optarg || *endptr != '\0' || - rowsPerInsert <= 0 || rowsPerInsert > INT_MAX || - errno == ERANGE) - { - pg_log_error("rows-per-insert must be in range %d..%d", - 1, INT_MAX); - exit_nicely(1); - } - dopt.dump_inserts = (int) rowsPerInsert; - break; - - case 11: /* include foreign data */ - simple_string_list_append(&foreign_servers_include_patterns, - optarg); - break; - - default: - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); - exit_nicely(1); - } + if (!process_option(c, optarg, &dopt, progname)) + exit_nicely(1); } + if (optsfilename) + read_options_from_file(optsfilename, + &dopt, + short_options, + long_options, + progname); + /* * Non-option argument specifies database name as long as it wasn't * already specified with -d / --dbname @@ -1049,6 +1092,7 @@ help(const char *progname) printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); printf(_(" --on-conflict-do-nothing add ON CONFLICT DO NOTHING to INSERT commands\n")); + printf(_(" --options-file=FILENAME read options from options file\n")); printf(_(" --quote-all-identifiers quote all identifiers, even if not key words\n")); printf(_(" --rows-per-insert=NROWS number of rows per INSERT; implies --inserts\n")); printf(_(" --section=SECTION dump named section (pre-data, data, or post-data)\n")); @@ -18635,3 +18679,380 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions, if (!res) pg_log_warning("could not parse reloptions array"); } + +/* + * Print error message and exit. + */ +static void +exit_invalid_optfile_format(FILE *fp, + char *message, + char *optname, + int optnamelen, + char *line, + int lineno) +{ + Assert(message); + + if (optnamelen > 0) + { + Assert(optname); + pg_log_error(message, optnamelen, optname, lineno); + } + else + pg_log_error(message, lineno); + + if (line) + fprintf(stderr, "LINE %d: %s\n", lineno, line); + + if (fp != stdin) + fclose(fp); + + exit_nicely(-1); +} + +/* + * Reads an option argument from file. Supports double qoutes + * bounded strings. In this case multi lines strings are supported. + */ +static void +read_optarg(FILE *fp, + char *filename, + char *str, + StringInfo line, + StringInfo optargument, + char *optname, + int optnamelen, + bool islongopt, + int *lineno) +{ + if (*str == '\0' || *str == '#') + { + if (islongopt) + exit_invalid_optfile_format(fp, + "option '--%.*s' requires an argument at line %d", + optname, optnamelen, + line->data, + *lineno); + else + exit_invalid_optfile_format(fp, + "option '-%.*s' requires an argument at line %d", + optname, optnamelen, + line->data, + *lineno); + } + + resetStringInfo(optargument); + + /* simple case */ + if (*str != '"') + { + char *start = str; + + /* read first white char */ + while (*str != '\0' && *str != '#') + { + if (*str == ' ') + break; + str++; + } + + appendBinaryStringInfo(optargument, start, str - start); + } + else + { + appendStringInfoChar(optargument, *str++); + + while (1) + { + if (*str == '\0') + { + /* multiline string, read next line */ + if (!pg_get_line_buf(fp, line)) + exit_invalid_optfile_format(fp, + "unexpected end of line at line %d", + NULL, 0, + NULL, + *lineno); + + if (ferror(fp)) + fatal("could not read from file \"%s\": %m", filename); + + appendStringInfoChar(optargument, '\n'); + + str = line->data; + (void) pg_strip_crlf(str); + *lineno += 1; + } + + appendStringInfoChar(optargument, *str); + + if (*str++ == '"') + { + if (*str == '"') + str++; + else + break; + } + } + } + + /* check garbage after optarg, but ignore white spaces */ + while (isspace(*str)) + str++; + + /* at the end there should be EOL or comment symbol */ + if (*str != '\0' && *str != '#') + exit_invalid_optfile_format(fp, + "unexpected characters after an option's argument at line %d", + NULL, 0, + line->data, + *lineno); +} + +/* + * Returns true, when long option is defined. Assign code, + * and check if option has required argument. + */ +static bool +is_recognized_longopt(const struct option *longopts, + char *optname, + int optnamelen, + int *opt, + bool *has_arg) +{ + int i; + + for (i = 0; longopts[i].name != NULL; i++) + { + if (strlen(longopts[i].name) == optnamelen && + strncmp(optname, longopts[i].name, optnamelen) == 0) + { + *has_arg = longopts[i].has_arg == required_argument; + + if (longopts[i].flag == NULL) + { + *opt = longopts[i].val; + + return true; + } + else + { + *longopts[i].flag = longopts[i].val; + *opt = 0; + + return true; + } + } + } + + return false; +} + +/* + * Returns true, when short option char is defined. + */ +static bool +is_valid_shortopt(const char *optstring, + char optname, + bool *has_arg) +{ + while (*optstring != '\0') + { + if (*optstring != ':' && *optstring == optname) + { + *has_arg = optstring[1] == ':'; + return true; + } + + optstring++; + } + + return false; +} + +/* + * Read dumped object specification from file + */ +static void +read_options_from_file(char *filename, + DumpOptions *dopt, + const char *optstring, + const struct option *longopts, + const char *progname) +{ + FILE *fp; + int lineno = 0; + StringInfoData line; + StringInfoData optargument; + + /* use "-" as symbol for stdin */ + if (strcmp(filename, "-") != 0) + { + fp = fopen(filename, "r"); + if (!fp) + fatal("could not open the input file \"%s\": %m", + filename); + } + else + fp = stdin; + + initStringInfo(&line); + initStringInfo(&optargument); + + while (pg_get_line_buf(fp, &line)) + { + char *optname; + char *str = line.data; + int opt; + int optnamelen; + bool has_arg; + + (void) pg_strip_crlf(str); + + lineno += 1; + + /* skip initial spaces */ + while (isspace(*str)) + str++; + + /* Ignore empty lines or lines with hash symbol (comment) */ + if (*str == '\0' || *str == '#') + continue; + + if (*str++ != '-') + exit_invalid_optfile_format(fp, + "non option arguments are not allowed in options file at line %d", + NULL, 0, + line.data, + lineno); + + if (*str == '-') + { + /* process long option */ + str++; + optname = str++; + + while (!isspace(*str) && *str != '=' && *str != '\0') + str++; + + optnamelen = str - optname; + + if (is_recognized_longopt(longopts, + optname, + optnamelen, + &opt, + &has_arg)) + { + /* skip optional spaces */ + while (isspace(*str)) + str++; + + /* + * Don't allow --options-file inside options file. + * It is simple protection against cycle. + */ + if (opt == OPTIONS_FILE_OPT_NUMBER) + exit_invalid_optfile_format(fp, + "option '%.*s' cannot be used in options file at line %d", + optname, optnamelen, + line.data, + lineno); + + if (has_arg) + { + /* skip optional = */ + if (*str == '=') + str++; + + /* skip optional spaces */ + while (isspace(*str)) + str++; + + read_optarg(fp, + filename, + str, + &line, + &optargument, + optname, + optnamelen, + true, + &lineno); + } + else + { + if (*str != '\0' && *str != '#') + exit_invalid_optfile_format(fp, + "option '--%.*s' doesn't allow an argument at line %d", + optname, optnamelen, + line.data, + lineno); + } + } + else + exit_invalid_optfile_format(fp, + "unrecognized option '--%.*s' at line %d", + optname, optnamelen, + line.data, + lineno); + } + else + { + /* process short option */ + optname = str++; + optnamelen = 1; + + /* skip optional spaces */ + while (isspace(*str)) + str++; + + if (is_valid_shortopt(optstring, *optname, &has_arg)) + { + if (has_arg) + { + read_optarg(fp, + filename, + str, + &line, + &optargument, + optname, + optnamelen, + false, + &lineno); + } + else + { + if (*str != '\0' && *str != '#') + exit_invalid_optfile_format(fp, + "option '-%.*s' doesn't allow an argument at line %d", + optname, optnamelen, + line.data, + lineno); + } + } + else + exit_invalid_optfile_format(fp, + "invalid option '-%.*s' at line %d", + optname, optnamelen, + line.data, + lineno); + + opt = *optname; + } + + if (opt != 0 && + !process_option(opt, optargument.data, dopt, progname)) + { + fclose(fp); + exit_nicely(-1); + } + } + + pfree(line.data); + pfree(optargument.data); + + if (ferror(fp)) + fatal("could not read from file \"%s\": %m", filename); + + if (fp != stdin) + fclose(fp); +} diff --git a/src/bin/pg_dump/t/004_pg_dump_optsfile.pl b/src/bin/pg_dump/t/004_pg_dump_optsfile.pl new file mode 100644 index 0000000000..0955e28a85 --- /dev/null +++ b/src/bin/pg_dump/t/004_pg_dump_optsfile.pl @@ -0,0 +1,165 @@ +use strict; +use warnings; + +use Config; +use PostgresNode; +use TestLib; +use Test::More tests => 30; + +my $tempdir = TestLib::tempdir; +my $inputfile; + + +my $node = get_new_node('main'); +my $port = $node->port; +my $backupdir = $node->backup_dir; +my $plainfile = "$backupdir/plain.sql"; + +$node->init; +$node->start; + +$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)"); +$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)"); +$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)"); +$node->safe_psql('postgres', "INSERT INTO table_one VALUES('*** TABLE ONE ***')"); +$node->safe_psql('postgres', "INSERT INTO table_two VALUES('*** TABLE TWO ***')"); +$node->safe_psql('postgres', "INSERT INTO table_three VALUES('*** TABLE THREE ***')"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; + +print $inputfile "-t table_one #comment\n"; +print $inputfile "-t table_two\n"; +print $inputfile "# skip this line\n"; +print $inputfile "\n"; +print $inputfile "--exclude-table-data=table_one\n"; +close $inputfile; + +my ($cmd, $stdout, $stderr, $result); +command_ok( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + "dump tables with filter"); + +my $dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public.table_one/m, "dumped table one"); +ok($dump =~ qr/^CREATE TABLE public.table_two/m, "dumped table two"); +ok($dump !~ qr/^CREATE TABLE public.table_three/m, "table three not dumped"); +ok($dump !~ qr/^COPY public.table_one/m, "content of table one is not included"); +ok($dump =~ qr/^COPY public.table_two/m, "content of table two is included"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; + +print $inputfile "-T table_one\n"; +close $inputfile; + +command_ok( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + "dump tables with filter"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE public.table_one/m, "table one not dumped"); +ok($dump =~ qr/^CREATE TABLE public.table_two/m, "dumped table two"); +ok($dump =~ qr/^CREATE TABLE public.table_three/m, "dumped table three"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; + +print $inputfile "-N public\n"; +close $inputfile; + +command_ok( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + "dump tables with filter"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE/m, "no table dumped"); + +######################################### +# For test of +f option we need created foreign server or accept +# fail and check error + +open $inputfile, '>', "$tempdir/inputfile.txt"; + +print $inputfile "--include-foreign-data doesnt_exists\n"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: no matching foreign servers were found for pattern/, + "dump foreign server"); + +######################################### +# Test broken input format + +open $inputfile, '>', "$tempdir/inputfile.txt"; +print $inputfile "k"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: non option arguments are not allowed in options file at line 1/, + "broken format check"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; +print $inputfile "-"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: invalid option '-' at line 1/, + "broken format check"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; +print $inputfile "-t"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: option '-t' requires an argument at line 1/, + "broken format check"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; +print $inputfile "-a someforeignserver"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: option '-a' doesn't allow an argument at line 1/, + "broken format check"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; +print $inputfile "-r"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: invalid option '-r' at line 1/, + "broken format check"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; +print $inputfile "--doesnt-exists"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: unrecognized option '--doesnt-exists' at line 1/, + "broken format check"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; +print $inputfile "--data-only badparameter"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: option '--data-only' doesn't allow an argument at line 1/, + "broken format check"); + +open $inputfile, '>', "$tempdir/inputfile.txt"; +print $inputfile "--table"; +close $inputfile; + +command_fails_like( + [ "pg_dump", '-p', $port, "-f", $plainfile, "--options-file=$tempdir/inputfile.txt", 'postgres' ], + qr/pg_dump: error: option '--table' requires an argument at line 1/, + "broken format check"); -- 2.17.0
>From ca431543d51e0630ba5b8f846ab23888ec7dc499 Mon Sep 17 00:00:00 2001 From: Justin Pryzby <pryz...@telsasoft.com> Date: Sat, 28 Nov 2020 17:00:40 -0600 Subject: [PATCH 2/4] Fix short options with double quotes --- src/bin/pg_dump/pg_dump.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 96acb8a1d5..990f890ff4 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -19000,6 +19000,7 @@ read_options_from_file(char *filename, /* process short option */ optname = str++; optnamelen = 1; + opt = *optname; /* skip optional spaces */ while (isspace(*str)) @@ -19035,8 +19036,6 @@ read_options_from_file(char *filename, optname, optnamelen, line.data, lineno); - - opt = *optname; } if (opt != 0 && -- 2.17.0
>From 98aff704e98e9228dc1e3e04e840785792611537 Mon Sep 17 00:00:00 2001 From: Justin Pryzby <pryz...@telsasoft.com> Date: Sat, 28 Nov 2020 16:58:59 -0600 Subject: [PATCH 3/4] typos --- src/bin/pg_dump/pg_dump.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 990f890ff4..43e3a022d5 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -316,9 +316,9 @@ static void read_options_from_file(char *filename, #define OPTIONS_FILE_OPT_NUMBER 12 /* - * It assign the values of options to related DumpOption fields or to - * some global values. It is called from twice. First, for processing - * the command line argumens. Second, for processing an options from + * It assigns the values of options to related DumpOption fields or to + * some global values. It is called twice. First, for processing + * the command line arguments. Second, for processing options from * options file. */ static bool @@ -18711,8 +18711,8 @@ exit_invalid_optfile_format(FILE *fp, } /* - * Reads an option argument from file. Supports double qoutes - * bounded strings. In this case multi lines strings are supported. + * Reads an option argument from file. Supports double-quoted + * strings. In this case multi-line strings are supported. */ static void read_optarg(FILE *fp, @@ -18796,7 +18796,7 @@ read_optarg(FILE *fp, } } - /* check garbage after optarg, but ignore white spaces */ + /* check garbage after optarg, but ignore white-space */ while (isspace(*str)) str++; -- 2.17.0
>From e569750c4ce6c6f13f601d2badf54d8df1b83fa5 Mon Sep 17 00:00:00 2001 From: Justin Pryzby <pryz...@telsasoft.com> Date: Sat, 28 Nov 2020 16:59:30 -0600 Subject: [PATCH 4/4] Allow option file to include options files FIXME: dopt is global and doesn't need to be passed around --- src/bin/pg_dump/pg_dump.c | 183 ++++++++++++++++++-------------------- 1 file changed, 86 insertions(+), 97 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 43e3a022d5..be6ba5d428 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -140,7 +140,6 @@ static char *use_role = NULL; static long rowsPerInsert; static int numWorkers = 1; static int compressLevel = -1; -static char *optsfilename = NULL; /* * The default number of rows per INSERT when @@ -308,13 +307,86 @@ static char *get_synchronized_snapshot(Archive *fout); static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); static void read_options_from_file(char *filename, - DumpOptions *dopt, const char *optstring, const struct option *longopts, const char *progname); #define OPTIONS_FILE_OPT_NUMBER 12 +static DumpOptions dopt; +static struct option long_options[] = { + {"data-only", no_argument, NULL, 'a'}, + {"blobs", no_argument, NULL, 'b'}, + {"no-blobs", no_argument, NULL, 'B'}, + {"clean", no_argument, NULL, 'c'}, + {"create", no_argument, NULL, 'C'}, + {"dbname", required_argument, NULL, 'd'}, + {"file", required_argument, NULL, 'f'}, + {"format", required_argument, NULL, 'F'}, + {"host", required_argument, NULL, 'h'}, + {"jobs", 1, NULL, 'j'}, + {"no-reconnect", no_argument, NULL, 'R'}, + {"no-owner", no_argument, NULL, 'O'}, + {"port", required_argument, NULL, 'p'}, + {"schema", required_argument, NULL, 'n'}, + {"exclude-schema", required_argument, NULL, 'N'}, + {"schema-only", no_argument, NULL, 's'}, + {"superuser", required_argument, NULL, 'S'}, + {"table", required_argument, NULL, 't'}, + {"exclude-table", required_argument, NULL, 'T'}, + {"no-password", no_argument, NULL, 'w'}, + {"password", no_argument, NULL, 'W'}, + {"username", required_argument, NULL, 'U'}, + {"verbose", no_argument, NULL, 'v'}, + {"no-privileges", no_argument, NULL, 'x'}, + {"no-acl", no_argument, NULL, 'x'}, + {"compress", required_argument, NULL, 'Z'}, + {"encoding", required_argument, NULL, 'E'}, + {"help", no_argument, NULL, '?'}, + {"version", no_argument, NULL, 'V'}, + + /* + * the following options don't have an equivalent short option letter + */ + {"attribute-inserts", no_argument, &dopt.column_inserts, 1}, + {"binary-upgrade", no_argument, &dopt.binary_upgrade, 1}, + {"column-inserts", no_argument, &dopt.column_inserts, 1}, + {"disable-dollar-quoting", no_argument, &dopt.disable_dollar_quoting, 1}, + {"disable-triggers", no_argument, &dopt.disable_triggers, 1}, + {"enable-row-security", no_argument, &dopt.enable_row_security, 1}, + {"exclude-table-data", required_argument, NULL, 4}, + {"extra-float-digits", required_argument, NULL, 8}, + {"if-exists", no_argument, &dopt.if_exists, 1}, + {"inserts", no_argument, NULL, 9}, + {"lock-wait-timeout", required_argument, NULL, 2}, + {"no-tablespaces", no_argument, &dopt.outputNoTablespaces, 1}, + {"quote-all-identifiers", no_argument, "e_all_identifiers, 1}, + {"load-via-partition-root", no_argument, &dopt.load_via_partition_root, 1}, + {"role", required_argument, NULL, 3}, + {"section", required_argument, NULL, 5}, + {"serializable-deferrable", no_argument, &dopt.serializable_deferrable, 1}, + {"snapshot", required_argument, NULL, 6}, + {"strict-names", no_argument, &strict_names, 1}, + {"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1}, + {"no-comments", no_argument, &dopt.no_comments, 1}, + {"no-publications", no_argument, &dopt.no_publications, 1}, + {"no-security-labels", no_argument, &dopt.no_security_labels, 1}, + {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, + {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, + {"no-subscriptions", no_argument, &dopt.no_subscriptions, 1}, + {"no-sync", no_argument, NULL, 7}, + {"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1}, + {"options-file", required_argument, NULL, OPTIONS_FILE_OPT_NUMBER}, + {"rows-per-insert", required_argument, NULL, 10}, + {"include-foreign-data", required_argument, NULL, 11}, + {"index-collation-versions-unknown", no_argument, &dopt.coll_unknown, 1}, + {"include-foreign-data-file", required_argument, NULL, 17}, + + {NULL, 0, NULL, 0} +}; + +const char *short_options = "abBcCd:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:"; + /* * It assigns the values of options to related DumpOption fields or to * some global values. It is called twice. First, for processing @@ -328,6 +400,7 @@ process_option(int opt, const char *progname) { char *endptr; + static int file_depth = 0; switch (opt) { @@ -512,7 +585,16 @@ process_option(int opt, break; case OPTIONS_FILE_OPT_NUMBER: /* filter implementation */ - optsfilename = pg_strdup(optargstr); + if (file_depth++ > 99) + { + pg_log_error("Don't include so many files including so many files"); + exit_nicely(1); + } + read_options_from_file(optargstr, + short_options, + long_options, + progname); + file_depth--; break; default: @@ -539,80 +621,6 @@ main(int argc, char **argv) int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; - static DumpOptions dopt; - - static struct option long_options[] = { - {"data-only", no_argument, NULL, 'a'}, - {"blobs", no_argument, NULL, 'b'}, - {"no-blobs", no_argument, NULL, 'B'}, - {"clean", no_argument, NULL, 'c'}, - {"create", no_argument, NULL, 'C'}, - {"dbname", required_argument, NULL, 'd'}, - {"file", required_argument, NULL, 'f'}, - {"format", required_argument, NULL, 'F'}, - {"host", required_argument, NULL, 'h'}, - {"jobs", 1, NULL, 'j'}, - {"no-reconnect", no_argument, NULL, 'R'}, - {"no-owner", no_argument, NULL, 'O'}, - {"port", required_argument, NULL, 'p'}, - {"schema", required_argument, NULL, 'n'}, - {"exclude-schema", required_argument, NULL, 'N'}, - {"schema-only", no_argument, NULL, 's'}, - {"superuser", required_argument, NULL, 'S'}, - {"table", required_argument, NULL, 't'}, - {"exclude-table", required_argument, NULL, 'T'}, - {"no-password", no_argument, NULL, 'w'}, - {"password", no_argument, NULL, 'W'}, - {"username", required_argument, NULL, 'U'}, - {"verbose", no_argument, NULL, 'v'}, - {"no-privileges", no_argument, NULL, 'x'}, - {"no-acl", no_argument, NULL, 'x'}, - {"compress", required_argument, NULL, 'Z'}, - {"encoding", required_argument, NULL, 'E'}, - {"help", no_argument, NULL, '?'}, - {"version", no_argument, NULL, 'V'}, - - /* - * the following options don't have an equivalent short option letter - */ - {"attribute-inserts", no_argument, &dopt.column_inserts, 1}, - {"binary-upgrade", no_argument, &dopt.binary_upgrade, 1}, - {"column-inserts", no_argument, &dopt.column_inserts, 1}, - {"disable-dollar-quoting", no_argument, &dopt.disable_dollar_quoting, 1}, - {"disable-triggers", no_argument, &dopt.disable_triggers, 1}, - {"enable-row-security", no_argument, &dopt.enable_row_security, 1}, - {"exclude-table-data", required_argument, NULL, 4}, - {"extra-float-digits", required_argument, NULL, 8}, - {"if-exists", no_argument, &dopt.if_exists, 1}, - {"inserts", no_argument, NULL, 9}, - {"lock-wait-timeout", required_argument, NULL, 2}, - {"no-tablespaces", no_argument, &dopt.outputNoTablespaces, 1}, - {"quote-all-identifiers", no_argument, "e_all_identifiers, 1}, - {"load-via-partition-root", no_argument, &dopt.load_via_partition_root, 1}, - {"role", required_argument, NULL, 3}, - {"section", required_argument, NULL, 5}, - {"serializable-deferrable", no_argument, &dopt.serializable_deferrable, 1}, - {"snapshot", required_argument, NULL, 6}, - {"strict-names", no_argument, &strict_names, 1}, - {"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1}, - {"no-comments", no_argument, &dopt.no_comments, 1}, - {"no-publications", no_argument, &dopt.no_publications, 1}, - {"no-security-labels", no_argument, &dopt.no_security_labels, 1}, - {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, - {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, - {"no-subscriptions", no_argument, &dopt.no_subscriptions, 1}, - {"no-sync", no_argument, NULL, 7}, - {"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1}, - {"options-file", required_argument, NULL, OPTIONS_FILE_OPT_NUMBER}, - {"rows-per-insert", required_argument, NULL, 10}, - {"include-foreign-data", required_argument, NULL, 11}, - {"index-collation-versions-unknown", no_argument, &dopt.coll_unknown, 1}, - {"include-foreign-data-file", required_argument, NULL, 17}, - - {NULL, 0, NULL, 0} - }; - - const char *short_options = "abBcCd:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:"; pg_logging_init(argv[0]); pg_logging_set_level(PG_LOG_WARNING); @@ -649,13 +657,6 @@ main(int argc, char **argv) exit_nicely(1); } - if (optsfilename) - read_options_from_file(optsfilename, - &dopt, - short_options, - long_options, - progname); - /* * Non-option argument specifies database name as long as it wasn't * already specified with -d / --dbname @@ -18875,7 +18876,6 @@ is_valid_shortopt(const char *optstring, */ static void read_options_from_file(char *filename, - DumpOptions *dopt, const char *optstring, const struct option *longopts, const char *progname) @@ -18947,17 +18947,6 @@ read_options_from_file(char *filename, while (isspace(*str)) str++; - /* - * Don't allow --options-file inside options file. - * It is simple protection against cycle. - */ - if (opt == OPTIONS_FILE_OPT_NUMBER) - exit_invalid_optfile_format(fp, - "option '%.*s' cannot be used in options file at line %d", - optname, optnamelen, - line.data, - lineno); - if (has_arg) { /* skip optional = */ @@ -19039,7 +19028,7 @@ read_options_from_file(char *filename, } if (opt != 0 && - !process_option(opt, optargument.data, dopt, progname)) + !process_option(opt, optargument.data, &dopt, progname)) { fclose(fp); exit_nicely(-1); -- 2.17.0