From bf371b501580cfa300c5198cc11ac67acc7631c3 Mon Sep 17 00:00:00 2001 From: Badrul Chowdhury Date: Mon, 25 Sep 2017 13:42:44 -0700 Subject: [PATCH] =?UTF-8?q?Introducing=20pgwire=20v3.1:=201.=20adds=20supp?= =?UTF-8?q?ort=20for=20backend=20(BE)=20to=20accept=20optional=20parameter?= =?UTF-8?q?s,=20ie=20parameters=20that=20have=20=E2=80=9C=5Fpq=5F=E2=80=9D?= =?UTF-8?q?=20as=20a=20proper=20prefix=20in=20their=20names=202.=20creates?= =?UTF-8?q?=20data=20structure=20for=20passing=20in=20additional=20paramet?= =?UTF-8?q?ers=20in=20FE,=20adding=20command=20line=20support=20is=20out?= =?UTF-8?q?=20of=20scope=20of=20this=20item=203.=20enhances=20FE->BE=20pro?= =?UTF-8?q?tocol=20negotiation:=20adds=20support=20for=20newer=20FE=20to?= =?UTF-8?q?=20connect=20to=20older=20BE=20while=20maintaining=20back-compa?= =?UTF-8?q?tibility,=20ie=20same=20FE=20<->=20BE,=20older=20FE=20to=20newe?= =?UTF-8?q?r=20BE=20work=20as=20before?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Makefile | 2 +- src/Makefile.global.in | 2 +- src/bin/pg_dump/pg_dump.c | 10 +-- src/bin/scripts/clusterdb.c | 2 +- src/bin/scripts/createuser.c | 2 +- src/bin/scripts/reindexdb.c | 6 +- src/bin/scripts/vacuumdb.c | 4 +- src/common/Makefile | 2 +- src/fe_utils/Makefile | 2 +- src/fe_utils/simple_list.c | 4 + src/include/fe_utils/simple_list.h | 1 + src/include/libpq/pqcomm.h | 2 +- src/interfaces/libpq/fe-connect.c | 173 +++++++++++++++++++++++++++++++++++- src/interfaces/libpq/fe-protocol3.c | 47 ++++++++-- src/interfaces/libpq/libpq-fe.h | 1 + src/interfaces/libpq/libpq-int.h | 9 +- src/tools/msvc/Mkvcbuild.pm | 2 +- 20 files changed, 372 insertions(+), 40 deletions(-) diff --git a/src/Makefile b/src/Makefile index 380da92c75..048d2d7ca8 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,10 +20,10 @@ SUBDIRS = \ backend/utils/mb/conversion_procs \ backend/snowball \ include \ + fe_utils \ interfaces \ backend/replication/libpqwalreceiver \ backend/replication/pgoutput \ - fe_utils \ bin \ pl \ makefiles \ diff --git a/src/Makefile.global.in b/src/Makefile.global.in index e8b3a519cb..c3b45ce650 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -480,7 +480,7 @@ endif # This macro is for use by libraries linking to libpq. (Because libpgport # isn't created with the same link flags as libpq, it can't be used.) -libpq = -L$(libpq_builddir) -lpq +libpq = -L$(libpq_builddir) -lpq -L$(top_builddir)/src/common -lpgcommon -L$(top_builddir)/src/fe_utils -lpgfeutils # This macro is for use by client executables (not libraries) that use libpq. # We force clients to pull symbols from the non-shared libraries libpgport diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 75f08cd792..9d3feb0a0a 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -110,16 +110,16 @@ static int strict_names = 0; * The string lists record the patterns given by command-line switches, * which we then convert to lists of OIDs of matching objects. */ -static SimpleStringList schema_include_patterns = {NULL, NULL}; +static SimpleStringList schema_include_patterns = {NULL, NULL, NULL}; static SimpleOidList schema_include_oids = {NULL, NULL}; -static SimpleStringList schema_exclude_patterns = {NULL, NULL}; +static SimpleStringList schema_exclude_patterns = {NULL, NULL, NULL}; static SimpleOidList schema_exclude_oids = {NULL, NULL}; -static SimpleStringList table_include_patterns = {NULL, NULL}; +static SimpleStringList table_include_patterns = {NULL, NULL, NULL}; static SimpleOidList table_include_oids = {NULL, NULL}; -static SimpleStringList table_exclude_patterns = {NULL, NULL}; +static SimpleStringList table_exclude_patterns = {NULL, NULL, NULL}; static SimpleOidList table_exclude_oids = {NULL, NULL}; -static SimpleStringList tabledata_exclude_patterns = {NULL, NULL}; +static SimpleStringList tabledata_exclude_patterns = {NULL, NULL, NULL}; static SimpleOidList tabledata_exclude_oids = {NULL, NULL}; diff --git a/src/bin/scripts/clusterdb.c b/src/bin/scripts/clusterdb.c index a6640aa57b..79fa46afd2 100644 --- a/src/bin/scripts/clusterdb.c +++ b/src/bin/scripts/clusterdb.c @@ -60,7 +60,7 @@ main(int argc, char *argv[]) bool quiet = false; bool alldb = false; bool verbose = false; - SimpleStringList tables = {NULL, NULL}; + SimpleStringList tables = {NULL, NULL, NULL}; progname = get_progname(argv[0]); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pgscripts")); diff --git a/src/bin/scripts/createuser.c b/src/bin/scripts/createuser.c index 0e36edcc5d..ff33ec63c3 100644 --- a/src/bin/scripts/createuser.c +++ b/src/bin/scripts/createuser.c @@ -58,7 +58,7 @@ main(int argc, char *argv[]) char *host = NULL; char *port = NULL; char *username = NULL; - SimpleStringList roles = {NULL, NULL}; + SimpleStringList roles = {NULL, NULL, NULL}; enum trivalue prompt_password = TRI_DEFAULT; bool echo = false; bool interactive = false; diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c index ffd611e7bb..2aede05156 100644 --- a/src/bin/scripts/reindexdb.c +++ b/src/bin/scripts/reindexdb.c @@ -68,9 +68,9 @@ main(int argc, char *argv[]) bool echo = false; bool quiet = false; bool verbose = false; - SimpleStringList indexes = {NULL, NULL}; - SimpleStringList tables = {NULL, NULL}; - SimpleStringList schemas = {NULL, NULL}; + SimpleStringList indexes = {NULL, NULL, NULL}; + SimpleStringList tables = {NULL, NULL, NULL}; + SimpleStringList schemas = {NULL, NULL, NULL}; progname = get_progname(argv[0]); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pgscripts")); diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index 5d2869ea6b..df77eccf5f 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -123,7 +123,7 @@ main(int argc, char *argv[]) vacuumingOptions vacopts; bool analyze_in_stages = false; bool alldb = false; - SimpleStringList tables = {NULL, NULL}; + SimpleStringList tables = {NULL, NULL, NULL}; int concurrentCons = 1; int tbl_count = 0; @@ -342,7 +342,7 @@ vacuum_one_database(const char *dbname, vacuumingOptions *vacopts, PGconn *conn; SimpleStringListCell *cell; ParallelSlot *slots = NULL; - SimpleStringList dbtables = {NULL, NULL}; + SimpleStringList dbtables = {NULL, NULL, NULL}; int i; bool failed = false; bool parallel = concurrentCons > 1; diff --git a/src/common/Makefile b/src/common/Makefile index 80e78d72fe..fa29c3e0ac 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -24,7 +24,7 @@ subdir = src/common top_builddir = ../.. include $(top_builddir)/src/Makefile.global -override CPPFLAGS := -DFRONTEND $(CPPFLAGS) +override CPPFLAGS := -DFRONTEND $(CPPFLAGS) -fPIC LIBS += $(PTHREAD_LIBS) # don't include subdirectory-path-dependent -I and -L switches diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile index ebce38ceb4..7706bc5fa9 100644 --- a/src/fe_utils/Makefile +++ b/src/fe_utils/Makefile @@ -17,7 +17,7 @@ subdir = src/fe_utils top_builddir = ../.. include $(top_builddir)/src/Makefile.global -override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS) +override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS) -fPIC OBJS = mbprint.o print.o psqlscan.o simple_list.o string_utils.o diff --git a/src/fe_utils/simple_list.c b/src/fe_utils/simple_list.c index 21a2e57297..0c26bead24 100644 --- a/src/fe_utils/simple_list.c +++ b/src/fe_utils/simple_list.c @@ -62,6 +62,10 @@ simple_oid_list_member(SimpleOidList *list, Oid val) void simple_string_list_append(SimpleStringList *list, const char *val) { + /* Cannot append to immutable list */ + if (list->is_immutable) + return; + SimpleStringListCell *cell; cell = (SimpleStringListCell *) diff --git a/src/include/fe_utils/simple_list.h b/src/include/fe_utils/simple_list.h index 97bb34f191..ea5e9af7b4 100644 --- a/src/include/fe_utils/simple_list.h +++ b/src/include/fe_utils/simple_list.h @@ -41,6 +41,7 @@ typedef struct SimpleStringList { SimpleStringListCell *head; SimpleStringListCell *tail; + bool is_immutable; } SimpleStringList; diff --git a/src/include/libpq/pqcomm.h b/src/include/libpq/pqcomm.h index 10c7434c41..3a8ea400ac 100644 --- a/src/include/libpq/pqcomm.h +++ b/src/include/libpq/pqcomm.h @@ -108,7 +108,7 @@ typedef struct /* The earliest and latest frontend/backend protocol version supported. */ #define PG_PROTOCOL_EARLIEST PG_PROTOCOL(2,0) -#define PG_PROTOCOL_LATEST PG_PROTOCOL(3,0) +#define PG_PROTOCOL_LATEST PG_PROTOCOL(3,1) typedef uint32 ProtocolVersion; /* FE/BE protocol version number */ diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index d0e97ecdd4..60ce6e797c 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -350,6 +350,11 @@ static const PQEnvironmentOption EnvironmentOptions[] = static const char uri_designator[] = "postgresql://"; static const char short_uri_designator[] = "postgres://"; +static const char *optional_parameter_prefix = "_pq_"; + +/* A list of string options to add as startup options */ +static SimpleStringList startup_parameters = {NULL, NULL, NULL}; + static bool connectOptions1(PGconn *conn, const char *conninfo); static bool connectOptions2(PGconn *conn); static int connectDBStart(PGconn *conn); @@ -2001,6 +2006,9 @@ PQconnectPoll(PGconn *conn) int optval; PQExpBufferData savedMessage; + /* Flag to check if newer FE is connecting to older BE. */ + bool server_is_older = false; + if (conn == NULL) return PGRES_POLLING_FAILED; @@ -2018,6 +2026,7 @@ PQconnectPoll(PGconn *conn) /* These are reading states */ case CONNECTION_AWAITING_RESPONSE: + case CONNECTION_NEGOTIATING: case CONNECTION_AUTH_OK: { /* Load waiting data */ @@ -2462,7 +2471,8 @@ keep_going: /* We will come back to here until there is */ if (PG_PROTOCOL_MAJOR(conn->pversion) >= 3) startpacket = pqBuildStartupPacket3(conn, &packetlen, - EnvironmentOptions); + EnvironmentOptions, + &startup_parameters); else startpacket = pqBuildStartupPacket2(conn, &packetlen, EnvironmentOptions); @@ -2649,6 +2659,12 @@ keep_going: /* We will come back to here until there is return PGRES_POLLING_READING; } + if (beresp == 'Y') + { + conn->status = CONNECTION_NEGOTIATING; + goto keep_going; + } + /* * Validate message type: we expect only an authentication * request or an error here. Anything else probably means @@ -2893,6 +2909,161 @@ keep_going: /* We will come back to here until there is goto keep_going; } + case CONNECTION_NEGOTIATING: + { + int minServerVersion; /* Min pgwire protocol supported by server */ + int maxServerVersion; /* Max pgwire protocol supported by server */ + int param_list_len; /* Length of list of parameters honored by server */ + int proper_prefix_len; /* Length of required prefix in optional parameter name */ + int i; /* Index for loop */ + PQExpBuffer buf; /* Buffer for data */ + char *rejected_param; /* Parameter that was rejected by the BE */ + int originalMsgLen; /* Length of message sans msg type */ + int runningMsgLength; /* Copy of originalMsgLen, will not be preserved */ + int available; /* Bytes available in message body */ + + /* Mark 'Y' as consumed: 1 byte for message type */ + conn->inCursor = conn->inStart + 1; + + /* + * Block until message length is read. + * + * No need to account for 2.x fixed-length message because this state cannot + * be reached by pre-3.0 server. + */ + if (pqGetInt(&originalMsgLen, sizeof(int32), conn)) + return PGRES_POLLING_READING; + + runningMsgLength = originalMsgLen; + + /* Block until each of the fields in the packet is read */ + if (pqGetInt(&minServerVersion, sizeof(int32), conn) || + pqGetInt(&maxServerVersion, sizeof(int32), conn) || + pqGetInt(¶m_list_len, sizeof(int32), conn)) + { + return PGRES_POLLING_READING; + } + + /* 4 bytes each for msgLength, min, max, and length of list of param names */ + runningMsgLength -= sizeof(int32) * 4; + + /* Create empty buffer */ + buf = createPQExpBuffer(); + for (i = 0; i < param_list_len; ++i) + { + if (pqGets(buf, conn)) + { + available = conn->inEnd - conn->inCursor; + if (available < runningMsgLength) + { + /* Enlarge buffer if required */ + if (pqCheckInBufferSpace(conn->inCursor + (size_t)runningMsgLength, conn)) + goto error_return; + + return PGRES_POLLING_READING; + } + else + { + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("encountered error while attempting " + "to read parameter name from connection\n")); + /* Free buffer */ + destroyPQExpBuffer(buf); + + goto error_return; + } + } + + /* Read string successfully, decrement message length to reflect this */ + runningMsgLength -= buf->len + 1; /* +1 for NULL */ + + simple_string_list_member(&startup_parameters, buf->data); + + /* pqGets() resets the buffer, so buffer is clean for next iteration */ + } + + /* Free buffer */ + destroyPQExpBuffer(buf); + + /* Check for extraneous data in packet, +1 to account for message type char */ + if (conn->inCursor != conn->inStart + 1 + originalMsgLen) + { + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("Extraneous data in ServerProtocolMessage packet\n")); + goto error_return; + } + + /* Mark read data consumed */ + conn->inStart = conn->inCursor; + + /* + * Emit error if FE is older than min supported by BE. + * + * This check will be effective once the minimum BE version is >= 3.0; + * otherwise, older FEs will be turned away when parsing startup packet. + */ + if (PG_PROTOCOL_MAJOR(conn->pversion) < PG_PROTOCOL_MAJOR(minServerVersion) || + (PG_PROTOCOL_MAJOR(conn->pversion) == PG_PROTOCOL_MAJOR(minServerVersion) && + PG_PROTOCOL_MINOR(conn->pversion) < PG_PROTOCOL_MINOR(minServerVersion))) + { + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("unsupported frontend protocol %u.%u: " + "server supports %u.%u to %u.%u"), + PG_PROTOCOL_MAJOR(conn->pversion), + PG_PROTOCOL_MINOR(conn->pversion), + PG_PROTOCOL_MAJOR(PG_PROTOCOL_EARLIEST), + PG_PROTOCOL_MINOR(PG_PROTOCOL_EARLIEST), + PG_PROTOCOL_MAJOR(PG_PROTOCOL_LATEST), + PG_PROTOCOL_MINOR(PG_PROTOCOL_LATEST)); + goto error_return; + } + + /* Set flag to enable parameter check if newer FE is connecting to older BE + * In this case, check if the non-optional parameters sent + * in StartupMsg by FE were accepted by the BE. + * + * If the FE version is the same as BE or it falls in the [min, max] + * range of BE, all the parameters sent by FE should be accepted by BE, + * so skip the accepted parameter check. + */ + else if (PG_PROTOCOL_MAJOR(conn->pversion) > PG_PROTOCOL_MAJOR(maxServerVersion) || + (PG_PROTOCOL_MAJOR(conn->pversion) == PG_PROTOCOL_MAJOR(maxServerVersion) && + PG_PROTOCOL_MINOR(conn->pversion) > PG_PROTOCOL_MINOR(maxServerVersion))) + { + server_is_older = true; + } + + /* + * Check whether all required parameters sent by newer FE were accepted by older BE. + * Do not error out if optional parameters were rejected by the BE. + */ + while (server_is_older && (rejected_param = simple_string_list_not_touched(&startup_parameters)) != NULL) + { + /* + * Terminate connection if the rejected parameter is not optional. + * This is not encoding-aware, which is okay because it is all on client-side + * optional_parameter_prefix must be a proper prefix of the rejected paramteter's name. + */ + proper_prefix_len = strlen(optional_parameter_prefix); + if (strlen(rejected_param) <= proper_prefix_len || + strncmp(rejected_param, optional_parameter_prefix, proper_prefix_len) != 0) + { + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("Non-optional startup parameter \'%s\'" + "was rejected by server\n"), + rejected_param); + goto error_return; + + } + + /* Mark member as visited */ + simple_string_list_member(&startup_parameters, rejected_param); + } + + conn->status = CONNECTION_AWAITING_RESPONSE; + goto keep_going; + } + case CONNECTION_AUTH_OK: { /* diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c index a484fe80a1..a3b841dc23 100644 --- a/src/interfaces/libpq/fe-protocol3.c +++ b/src/interfaces/libpq/fe-protocol3.c @@ -22,6 +22,12 @@ #include "mb/pg_wchar.h" +/* + * Simple string list data structure to track + * startup parameters that were accepted. + */ +#include "fe_utils/simple_list.h" + #ifdef WIN32 #include "win32.h" #else @@ -54,7 +60,8 @@ static int getReadyForQuery(PGconn *conn); static void reportErrorPosition(PQExpBuffer msg, const char *query, int loc, int encoding); static int build_startup_packet(const PGconn *conn, char *packet, - const PQEnvironmentOption *options); + const PQEnvironmentOption *options, + SimpleStringList *startup_parameters); /* @@ -2116,15 +2123,16 @@ pqFunctionCall3(PGconn *conn, Oid fnid, */ char * pqBuildStartupPacket3(PGconn *conn, int *packetlen, - const PQEnvironmentOption *options) + const PQEnvironmentOption *options, + SimpleStringList *startup_parameters) { char *startpacket; - *packetlen = build_startup_packet(conn, NULL, options); + *packetlen = build_startup_packet(conn, NULL, options, startup_parameters); startpacket = (char *) malloc(*packetlen); if (!startpacket) return NULL; - *packetlen = build_startup_packet(conn, startpacket, options); + *packetlen = build_startup_packet(conn, startpacket, options, startup_parameters); return startpacket; } @@ -2139,11 +2147,15 @@ pqBuildStartupPacket3(PGconn *conn, int *packetlen, */ static int build_startup_packet(const PGconn *conn, char *packet, - const PQEnvironmentOption *options) + const PQEnvironmentOption *options, + SimpleStringList *startup_parameters) { int packet_len = 0; const PQEnvironmentOption *next_eo; const char *val; + int nCells = 0; + char *name, *value; + SimpleStringListCell *pCurr; /* Protocol version comes first. */ if (packet) @@ -2195,6 +2207,31 @@ build_startup_packet(const PGconn *conn, char *packet, } } + if (startup_parameters) + { + for (pCurr = startup_parameters->head; pCurr != NULL; pCurr = pCurr->next, ++nCells) + { + if ((nCells % 2) == 0) + name = pCurr->val; + else + { + value = pCurr->val; + ADD_STARTUP_OPTION(name, value); + + /* Mark value consumed */ + pCurr->touched = true; + } + } + + if ((nCells % 2) != 0) + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("at least one parameter specified in" + "StartupMessage does not have a value\n")); + pqSaveErrorResult(conn); + } + } + /* Add trailing terminator */ if (packet) packet[packet_len] = '\0'; diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h index 1d915e7915..d04f48544c 100644 --- a/src/interfaces/libpq/libpq-fe.h +++ b/src/interfaces/libpq/libpq-fe.h @@ -58,6 +58,7 @@ typedef enum CONNECTION_MADE, /* Connection OK; waiting to send. */ CONNECTION_AWAITING_RESPONSE, /* Waiting for a response from the * postmaster. */ + CONNECTION_NEGOTIATING, /* Negotiating pgwire protocol between FE/BE */ CONNECTION_AUTH_OK, /* Received authentication; waiting for * backend startup. */ CONNECTION_SETENV, /* Negotiating environment. */ diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 42913604e3..10121075f2 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -43,6 +43,12 @@ /* include stuff found in fe only */ #include "pqexpbuffer.h" +/* + * Simple string list data structure to track + * startup parameters that were accepted. + */ +#include "fe_utils/simple_list.h" + #ifdef ENABLE_GSS #if defined(HAVE_GSSAPI_H) #include @@ -598,7 +604,8 @@ extern PGresult *pqFunctionCall2(PGconn *conn, Oid fnid, /* === in fe-protocol3.c === */ extern char *pqBuildStartupPacket3(PGconn *conn, int *packetlen, - const PQEnvironmentOption *options); + const PQEnvironmentOption *options, + SimpleStringList *parameters); extern void pqParseInput3(PGconn *conn); extern int pqGetErrorNotice3(PGconn *conn, bool isError); extern void pqBuildErrorMessage3(PQExpBuffer msg, const PGresult *res, diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 686c7369f6..a440ca303f 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -234,7 +234,7 @@ sub mkvcbuild $libpq->UseDef('src/interfaces/libpq/libpqdll.def'); $libpq->ReplaceFile('src/interfaces/libpq/libpqrc.c', 'src/interfaces/libpq/libpq.rc'); - $libpq->AddReference($libpgport); + $libpq->AddReference($libpgcommon, $libpgfeutils, $libpgport); # The OBJS scraper doesn't know about ifdefs, so remove fe-secure-openssl.c # and sha2_openssl.c if building without OpenSSL, and remove sha2.c if -- 2.13.2.windows.1