On Sat, Nov 30, 2019 at 9:15 PM Michael Paquier <mich...@paquier.xyz> wrote: > > On Thu, Oct 03, 2019 at 11:39:37AM -0700, Andres Freund wrote: > > Color me unconvinced. > > The latest comments of the thread have not been addressed yet. so I am > marking the patch as returned with feedback. If you think that's not > correct, please feel free to update the status of the patch. If you > do so, please provide at the same time a rebased version of the patch, > as it is failing to apply on HEAD.
I've finally had some time to update the patchset with an attempt at addressing Andres' concerns. Note that the current spin has a bug which does not allow syslogger to run properly. Additionally, it is squashed into one patch again. I intend to split the patch out into separate functional patches and fix the syslogger issue, but wanted to get this on the ticket for the open CommitFest before it closes. I'll go ahead and re-add it and will be pushing updates as I have them. I intend to give this patchset the time it deserves, so will be much more responsive this go-around. Thanks, -- Mike Palmiotto https://crunchydata.com
From 34feb7399fe218eab0da851032299f81e7ad6689 Mon Sep 17 00:00:00 2001 From: Mike Palmiotto <mike.palmio...@crunchydata.com> Date: Tue, 19 Feb 2019 15:29:33 +0000 Subject: [PATCH 2/2] Add a hook to allow extensions to set worker metadata This patch implements a hook that allows extensions to modify a worker process' metadata in backends. Additional work done by Yuli Khodorkovskiy <y...@crunchydata.com> Original discussion here: https://www.postgresql.org/message-id/flat/CAMN686FE0OdZKp9YPO%3DhtC6LnA6aW4r-%2Bjq%3D3Q5RAoFQgW8EtA%40mail.gmail.com --- src/backend/postmaster/fork_process.c | 11 +++++++++++ src/include/postmaster/fork_process.h | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/src/backend/postmaster/fork_process.c b/src/backend/postmaster/fork_process.c index def3cee37e..e392f5207e 100644 --- a/src/backend/postmaster/fork_process.c +++ b/src/backend/postmaster/fork_process.c @@ -22,6 +22,14 @@ #include "postmaster/fork_process.h" +/* + * This hook allows plugins to get control of the child process following + * a successful fork_process(). It can be used to perform some action in + * extensions to set metadata in backends (including special backends) upon + * setting of the session user. + */ +ForkProcess_post_hook_type ForkProcess_post_hook = NULL; + #ifndef WIN32 /* * Wrapper for fork(). Return values are the same as those for fork(): @@ -114,6 +122,9 @@ fork_process(void) #ifdef USE_OPENSSL RAND_cleanup(); #endif + + if (ForkProcess_post_hook) + (*ForkProcess_post_hook) (); } return result; diff --git a/src/include/postmaster/fork_process.h b/src/include/postmaster/fork_process.h index a42f859a08..b4d1560b72 100644 --- a/src/include/postmaster/fork_process.h +++ b/src/include/postmaster/fork_process.h @@ -20,4 +20,8 @@ extern pid_t fork_process(void); extern pid_t postmaster_forkexec(int argc, char *argvp[]); #endif +/* Hook for plugins to get control after a successful fork_process() */ +typedef void (*ForkProcess_post_hook_type) (); +extern PGDLLIMPORT ForkProcess_post_hook_type ForkProcess_post_hook; + #endif /* FORK_PROCESS_H */ -- 2.21.0
From 6fb91ba335153882d16d2dff28d584c3c0bc83a2 Mon Sep 17 00:00:00 2001 From: Mike Palmiotto <mike.palmio...@crunchydata.com> Date: Fri, 27 Sep 2019 12:28:19 -0400 Subject: [PATCH 1/2] Introduce subprocess infrastructure This is an initial attempt at coalescing all of the postmaster subprocess startups into one central framework. The patchset introduces a MySubprocess global, which contains an PgSubprocess entry from the process_types array. Each entry has defined entrypoints, cleanup functions, prep functions, and a few other control-oriented variables. This is a rework of https://commitfest.postgresql.org/25/2259/, which attempts to address the feedback from Andres Freund. --- src/backend/bootstrap/bootstrap.c | 78 +- src/backend/main/main.c | 3 + src/backend/postmaster/Makefile | 1 + src/backend/postmaster/autovacuum.c | 165 +--- src/backend/postmaster/bgworker.c | 151 +++- src/backend/postmaster/bgwriter.c | 2 +- src/backend/postmaster/checkpointer.c | 2 +- src/backend/postmaster/pgarch.c | 83 +-- src/backend/postmaster/pgstat.c | 89 +-- src/backend/postmaster/postmaster.c | 786 +++++++------------- src/backend/postmaster/startup.c | 8 +- src/backend/postmaster/subprocess.c | 206 +++++ src/backend/postmaster/syslogger.c | 278 +++---- src/backend/postmaster/walwriter.c | 2 +- src/backend/replication/walreceiver.c | 2 +- src/backend/utils/init/globals.c | 3 +- src/backend/utils/init/postinit.c | 2 +- src/include/bootstrap/bootstrap.h | 4 +- src/include/miscadmin.h | 1 + src/include/pgstat.h | 5 +- src/include/postmaster/autovacuum.h | 13 +- src/include/postmaster/bgworker_internals.h | 7 +- src/include/postmaster/bgwriter.h | 4 +- src/include/postmaster/fork_process.h | 6 + src/include/postmaster/pgarch.h | 9 +- src/include/postmaster/postmaster.h | 62 +- src/include/postmaster/startup.h | 5 +- src/include/postmaster/subprocess.h | 62 ++ src/include/postmaster/syslogger.h | 6 +- src/include/postmaster/walwriter.h | 2 +- src/include/replication/walreceiver.h | 2 +- 31 files changed, 989 insertions(+), 1060 deletions(-) create mode 100644 src/backend/postmaster/subprocess.c create mode 100644 src/include/postmaster/subprocess.h diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index bfc629c753..ed1066efd0 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -35,6 +35,7 @@ #include "pgstat.h" #include "postmaster/bgwriter.h" #include "postmaster/startup.h" +#include "postmaster/subprocess.h" #include "postmaster/walwriter.h" #include "replication/walreceiver.h" #include "storage/bufmgr.h" @@ -56,8 +57,6 @@ uint32 bootstrap_data_checksum_version = 0; /* No checksum */ #define ALLOC(t, c) \ ((t *) MemoryContextAllocZero(TopMemoryContext, (unsigned)(c) * sizeof(t))) -static void CheckerModeMain(void); -static void BootstrapModeMain(void); static void bootstrap_signals(void); static void ShutdownAuxiliaryProcess(int code, Datum arg); static Form_pg_attribute AllocateAttribute(void); @@ -426,56 +425,18 @@ AuxiliaryProcessMain(int argc, char *argv[]) */ SetProcessingMode(NormalProcessing); - switch (MyAuxProcType) - { - case CheckerProcess: - /* don't set signals, they're useless here */ - CheckerModeMain(); - proc_exit(1); /* should never return */ - - case BootstrapProcess: - - /* - * There was a brief instant during which mode was Normal; this is - * okay. We need to be in bootstrap mode during BootStrapXLOG for - * the sake of multixact initialization. - */ - SetProcessingMode(BootstrapProcessing); - bootstrap_signals(); - BootStrapXLOG(); - BootstrapModeMain(); - proc_exit(1); /* should never return */ - - case StartupProcess: - /* don't set signals, startup process has its own agenda */ - StartupProcessMain(); - proc_exit(1); /* should never return */ - - case BgWriterProcess: - /* don't set signals, bgwriter has its own agenda */ - BackgroundWriterMain(); - proc_exit(1); /* should never return */ - - case CheckpointerProcess: - /* don't set signals, checkpointer has its own agenda */ - CheckpointerMain(); - proc_exit(1); /* should never return */ - - case WalWriterProcess: - /* don't set signals, walwriter has its own agenda */ - InitXLOGAccess(); - WalWriterMain(); - proc_exit(1); /* should never return */ - - case WalReceiverProcess: - /* don't set signals, walreceiver has its own agenda */ - WalReceiverMain(); - proc_exit(1); /* should never return */ - - default: - elog(PANIC, "unrecognized process type: %d", (int) MyAuxProcType); - proc_exit(1); - } + /* + * At this point, we should know what kind of process we are. If the + * MySubprocess global has not been initialized, just cast and pass along the + * MyAuxProcType enum + */ + if (MySubprocess == NULL) + InitializeMySubprocess((SubprocessType)MyAuxProcType); + + /* Now jump into the subprocess main function and never look back! */ + MySubprocess->entrypoint(argc, argv); + + proc_exit(1); /* should never return */ } /* @@ -484,8 +445,8 @@ AuxiliaryProcessMain(int argc, char *argv[]) * settings). Since, in fact, that was already done by BaseInit(), * we have nothing more to do here. */ -static void -CheckerModeMain(void) +void +CheckerModeMain(int argc, char *argv[]) { proc_exit(0); } @@ -497,11 +458,15 @@ CheckerModeMain(void) * The bootstrap backend doesn't speak SQL, but instead expects * commands in a special bootstrap language. */ -static void -BootstrapModeMain(void) +void +BootstrapModeMain(int argc, char *argv[]) { int i; + SetProcessingMode(BootstrapProcessing); + bootstrap_signals(); + BootStrapXLOG(); + Assert(!IsUnderPostmaster); Assert(IsBootstrapProcessingMode()); @@ -516,7 +481,6 @@ BootstrapModeMain(void) * Do backend-like initialization for bootstrap mode */ InitProcess(); - InitPostgres(NULL, InvalidOid, NULL, InvalidOid, NULL, false); /* Initialize stuff for bootstrap-file processing */ diff --git a/src/backend/main/main.c b/src/backend/main/main.c index da3dae9e25..d1e8089098 100644 --- a/src/backend/main/main.c +++ b/src/backend/main/main.c @@ -35,6 +35,7 @@ #include "common/username.h" #include "port/atomics.h" #include "postmaster/postmaster.h" +#include "postmaster/subprocess.h" #include "storage/s_lock.h" #include "storage/spin.h" #include "tcop/tcopprot.h" @@ -199,7 +200,9 @@ main(int argc, char *argv[]) #endif if (argc > 1 && strcmp(argv[1], "--boot") == 0) + { AuxiliaryProcessMain(argc, argv); /* does not return */ + } else if (argc > 1 && strcmp(argv[1], "--describe-config") == 0) GucInfoMain(); /* does not return */ else if (argc > 1 && strcmp(argv[1], "--single") == 0) diff --git a/src/backend/postmaster/Makefile b/src/backend/postmaster/Makefile index bfdf6a833d..52b4c2142c 100644 --- a/src/backend/postmaster/Makefile +++ b/src/backend/postmaster/Makefile @@ -23,6 +23,7 @@ OBJS = \ pgstat.o \ postmaster.o \ startup.o \ + subprocess.o \ syslogger.o \ walwriter.o diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index e3a43d3296..cb55fb9019 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -83,7 +83,6 @@ #include "nodes/makefuncs.h" #include "pgstat.h" #include "postmaster/autovacuum.h" -#include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" #include "storage/bufmgr.h" @@ -135,8 +134,8 @@ int Log_autovacuum_min_duration = -1; #define MAX_AUTOVAC_SLEEPTIME 300 /* seconds */ /* Flags to tell if we are in an autovacuum process */ -static bool am_autovacuum_launcher = false; -static bool am_autovacuum_worker = false; +bool am_autovacuum_launcher = false; +bool am_autovacuum_worker = false; /* Flags set by signal handlers */ static volatile sig_atomic_t got_SIGUSR2 = false; @@ -302,13 +301,6 @@ static WorkerInfo MyWorkerInfo = NULL; /* PID of launcher, valid only in worker while shutting down */ int AutovacuumLauncherPid = 0; -#ifdef EXEC_BACKEND -static pid_t avlauncher_forkexec(void); -static pid_t avworker_forkexec(void); -#endif -NON_EXEC_STATIC void AutoVacWorkerMain(int argc, char *argv[]) pg_attribute_noreturn(); -NON_EXEC_STATIC void AutoVacLauncherMain(int argc, char *argv[]) pg_attribute_noreturn(); - static Oid do_start_worker(void); static void HandleAutoVacLauncherInterrupts(void); static void AutoVacLauncherShutdown(void) pg_attribute_noreturn(); @@ -346,88 +338,15 @@ static void autovac_report_workitem(AutoVacuumWorkItem *workitem, static void avl_sigusr2_handler(SIGNAL_ARGS); static void autovac_refresh_stats(void); - - /******************************************************************** * AUTOVACUUM LAUNCHER CODE ********************************************************************/ -#ifdef EXEC_BACKEND -/* - * forkexec routine for the autovacuum launcher process. - * - * Format up the arglist, then fork and exec. - */ -static pid_t -avlauncher_forkexec(void) -{ - char *av[10]; - int ac = 0; - - av[ac++] = "postgres"; - av[ac++] = "--forkavlauncher"; - av[ac++] = NULL; /* filled in by postmaster_forkexec */ - av[ac] = NULL; - - Assert(ac < lengthof(av)); - - return postmaster_forkexec(ac, av); -} - -/* - * We need this set from the outside, before InitProcess is called - */ -void -AutovacuumLauncherIAm(void) -{ - am_autovacuum_launcher = true; -} -#endif - -/* - * Main entry point for autovacuum launcher process, to be called from the - * postmaster. - */ -int -StartAutoVacLauncher(void) -{ - pid_t AutoVacPID; - -#ifdef EXEC_BACKEND - switch ((AutoVacPID = avlauncher_forkexec())) -#else - switch ((AutoVacPID = fork_process())) -#endif - { - case -1: - ereport(LOG, - (errmsg("could not fork autovacuum launcher process: %m"))); - return 0; - -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - - AutoVacLauncherMain(0, NULL); - break; -#endif - default: - return (int) AutoVacPID; - } - - /* shouldn't get here */ - return 0; -} - /* * Main loop for the autovacuum launcher process. */ -NON_EXEC_STATIC void -AutoVacLauncherMain(int argc, char *argv[]) +void +AutoVacLauncherMain(pg_attribute_unused() int argc, pg_attribute_unused() char *argv[]) { sigjmp_buf local_sigjmp_buf; @@ -1423,83 +1342,11 @@ avl_sigusr2_handler(SIGNAL_ARGS) * AUTOVACUUM WORKER CODE ********************************************************************/ -#ifdef EXEC_BACKEND -/* - * forkexec routines for the autovacuum worker. - * - * Format up the arglist, then fork and exec. - */ -static pid_t -avworker_forkexec(void) -{ - char *av[10]; - int ac = 0; - - av[ac++] = "postgres"; - av[ac++] = "--forkavworker"; - av[ac++] = NULL; /* filled in by postmaster_forkexec */ - av[ac] = NULL; - - Assert(ac < lengthof(av)); - - return postmaster_forkexec(ac, av); -} - -/* - * We need this set from the outside, before InitProcess is called - */ -void -AutovacuumWorkerIAm(void) -{ - am_autovacuum_worker = true; -} -#endif - -/* - * Main entry point for autovacuum worker process. - * - * This code is heavily based on pgarch.c, q.v. - */ -int -StartAutoVacWorker(void) -{ - pid_t worker_pid; - -#ifdef EXEC_BACKEND - switch ((worker_pid = avworker_forkexec())) -#else - switch ((worker_pid = fork_process())) -#endif - { - case -1: - ereport(LOG, - (errmsg("could not fork autovacuum worker process: %m"))); - return 0; - -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - - AutoVacWorkerMain(0, NULL); - break; -#endif - default: - return (int) worker_pid; - } - - /* shouldn't get here */ - return 0; -} - /* * AutoVacWorkerMain */ -NON_EXEC_STATIC void -AutoVacWorkerMain(int argc, char *argv[]) +void +AutoVacWorkerMain(pg_attribute_unused() int argc, pg_attribute_unused() char *argv[]) { sigjmp_buf local_sigjmp_buf; Oid dbid; diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 75fc0d5d33..acda30be40 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -33,8 +33,10 @@ #include "storage/shmem.h" #include "tcop/tcopprot.h" #include "utils/ascii.h" +#include "utils/memutils.h" #include "utils/ps_status.h" #include "utils/timeout.h" +#include "utils/timestamp.h" /* * The postmaster's list of registered background workers, in private memory. @@ -133,7 +135,97 @@ static const struct /* Private functions. */ static bgworker_main_type LookupBackgroundWorkerFunction(const char *libraryname, const char *funcname); +static bool assign_backendlist_entry(RegisteredBgWorker *rw); +bool BackgroundWorkerCleanup(int argc, char *argv[]); +void BackgroundWorkerParentMain(int argc, char *argv[]); + +/* + * Allocate the Backend struct for a connected background worker, but don't + * add it to the list of backends just yet. + * + * On failure, return false without changing any worker state. + * + * Some info from the Backend is copied into the passed rw. + */ +static bool +assign_backendlist_entry(RegisteredBgWorker *rw) +{ + Backend *bn; + + /* + * Compute the cancel key that will be assigned to this session. We + * probably don't need cancel keys for background workers, but we'd better + * have something random in the field to prevent unfriendly people from + * sending cancels to them. + */ + if (!RandomCancelKey(&MyCancelKey)) + { + ereport(LOG, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random cancel key"))); + return false; + } + + bn = malloc(sizeof(Backend)); + if (bn == NULL) + { + ereport(LOG, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); + return false; + } + + bn->cancel_key = MyCancelKey; + bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); + bn->bkend_type = BACKEND_TYPE_BGWORKER; + bn->dead_end = false; + bn->bgworker_notify = false; + + rw->rw_backend = bn; + rw->rw_child_slot = bn->child_slot; + + return true; +} + +/* + * BgWorkerPrep + * + * Postmaster subroutine to prepare a bgworker subprocess. + * + * Returns 0 on success or -1 on failure. + * + * Note: if fail, we will be called again from the postmaster main loop. + */ +int +BgWorkerPrep(int argc, char *argv[]) +{ + RegisteredBgWorker *rw = CurrentBgWorker; + + Assert(CurrentBgWorker); + Assert(rw->rw_pid == 0); + + /* + * Allocate and assign the Backend element. Note we must do this before + * forking, so that we can handle out of memory properly. + * + * Treat failure as though the worker had crashed. That way, the + * postmaster will wait a bit before attempting to start it again; if it + * tried again right away, most likely it'd find itself repeating the + * out-of-memory or fork failure condition. + */ + if (!assign_backendlist_entry(rw)) + { + rw->rw_crashed_at = GetCurrentTimestamp(); + return -1; + } + + ereport(DEBUG1, + (errmsg("starting background worker process \"%s\"", + rw->rw_worker.bgw_name))); + + return 0; +} /* * Calculate shared memory needed. @@ -677,12 +769,31 @@ bgworker_sigusr1_handler(SIGNAL_ARGS) * postmaster. */ void -StartBackgroundWorker(void) +BackgroundWorkerMain(pg_attribute_unused() int argc, pg_attribute_unused() char *argv[]) { sigjmp_buf local_sigjmp_buf; - BackgroundWorker *worker = MyBgworkerEntry; + BackgroundWorker *worker; bgworker_main_type entrypt; + /* Close the postmaster's sockets */ + ClosePostmasterPorts(); + + /* + * Before blowing away PostmasterContext, save this bgworker's + * data where it can find it. + */ + MyBgworkerEntry = (BackgroundWorker *) + MemoryContextAlloc(TopMemoryContext, sizeof(BackgroundWorker)); + memcpy(MyBgworkerEntry, &(CurrentBgWorker->rw_worker), sizeof(BackgroundWorker)); + + worker = (BackgroundWorker *) MemoryContextAlloc(TopMemoryContext, sizeof(BackgroundWorker)); + MyBgworkerEntry = worker; + + /* Now we can release postmaster's working memory context */ + MemoryContextSwitchTo(TopMemoryContext); + MemoryContextDelete(PostmasterContext); + PostmasterContext = NULL; + if (worker == NULL) elog(FATAL, "unable to find bgworker entry"); @@ -816,6 +927,42 @@ StartBackgroundWorker(void) proc_exit(0); } +bool +BackgroundWorkerCleanup(pg_attribute_unused() int argc, pg_attribute_unused() char *argv[]) +{ + RegisteredBgWorker *rw = CurrentBgWorker; + + /* in postmaster, fork failed ... */ + ereport(LOG, + (errmsg("could not fork worker process: %m"))); + + /* undo what assign_backendlist_entry did */ + ReleasePostmasterChildSlot(rw->rw_child_slot); + rw->rw_child_slot = 0; + free(rw->rw_backend); + rw->rw_backend = NULL; + /* mark entry as crashed, so we'll try again later */ + rw->rw_crashed_at = GetCurrentTimestamp(); + + /* Don't panic. */ + return false; +} + +void +BackgroundWorkerParentMain(pg_attribute_unused() int argc, pg_attribute_unused() char *argv[]) +{ + RegisteredBgWorker *rw = CurrentBgWorker; + + rw->rw_pid = MyChildProcPid; + rw->rw_backend->pid = rw->rw_pid; + ReportBackgroundWorkerPID(rw); + /* add new worker to lists of backends */ + dlist_push_head(&BackendList, &(rw->rw_backend->elem)); +#ifdef EXEC_BACKEND + ShmemBackendArrayAdd(rw->rw_backend); +#endif +} + /* * Register a new static background worker. * diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index 069e27e427..b2170938f3 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -91,7 +91,7 @@ static XLogRecPtr last_snapshot_lsn = InvalidXLogRecPtr; * basic execution environment, but not enabled signals yet. */ void -BackgroundWriterMain(void) +BackgroundWriterMain(int argc, char *argv[]) { sigjmp_buf local_sigjmp_buf; MemoryContext bgwriter_context; diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index e354a78725..83f7be1b1e 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -180,7 +180,7 @@ static void ReqCheckpointHandler(SIGNAL_ARGS); * basic execution environment, but not enabled signals yet. */ void -CheckpointerMain(void) +CheckpointerMain(int argc, char *argv[]) { sigjmp_buf local_sigjmp_buf; MemoryContext checkpointer_context; diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 3ca30badb2..02c98d5d9f 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -91,11 +91,7 @@ static volatile sig_atomic_t ready_to_stop = false; * Local function forward declarations * ---------- */ -#ifdef EXEC_BACKEND -static pid_t pgarch_forkexec(void); -#endif - -NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn(); +void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn(); static void pgarch_exit(SIGNAL_ARGS); static void pgarch_waken(SIGNAL_ARGS); static void pgarch_waken_stop(SIGNAL_ARGS); @@ -112,7 +108,7 @@ static void pgarch_archiveDone(char *xlog); */ /* - * pgarch_start + * PgArchiverPrep * * Called from postmaster at startup or after an existing archiver * died. Attempt to fire up a fresh archiver process. @@ -122,10 +118,9 @@ static void pgarch_archiveDone(char *xlog); * Note: if fail, we will be called again from the postmaster main loop. */ int -pgarch_start(void) +PgArchiverPrep(int argc, char *argv[]) { time_t curtime; - pid_t pgArchPid; /* * Do nothing if no archiver needed @@ -140,43 +135,10 @@ pgarch_start(void) * the postmaster main loop, we will get another chance later. */ curtime = time(NULL); - if ((unsigned int) (curtime - last_pgarch_start_time) < + if ((unsigned int) (curtime - last_pgarch_start_time) >= (unsigned int) PGARCH_RESTART_INTERVAL) - return 0; - last_pgarch_start_time = curtime; - -#ifdef EXEC_BACKEND - switch ((pgArchPid = pgarch_forkexec())) -#else - switch ((pgArchPid = fork_process())) -#endif - { - case -1: - ereport(LOG, - (errmsg("could not fork archiver: %m"))); - return 0; - -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - - /* Drop our connection to postmaster's shared memory, as well */ - dsm_detach_all(); - PGSharedMemoryDetach(); - - PgArchiverMain(0, NULL); - break; -#endif - - default: - return (int) pgArchPid; - } + last_pgarch_start_time = curtime; - /* shouldn't get here */ return 0; } @@ -186,42 +148,19 @@ pgarch_start(void) */ -#ifdef EXEC_BACKEND - -/* - * pgarch_forkexec() - - * - * Format up the arglist for, then fork and exec, archive process - */ -static pid_t -pgarch_forkexec(void) -{ - char *av[10]; - int ac = 0; - - av[ac++] = "postgres"; - - av[ac++] = "--forkarch"; - - av[ac++] = NULL; /* filled in by postmaster_forkexec */ - - av[ac] = NULL; - Assert(ac < lengthof(av)); - - return postmaster_forkexec(ac, av); -} -#endif /* EXEC_BACKEND */ - - /* * PgArchiverMain * * The argc/argv parameters are valid only in EXEC_BACKEND case. However, * since we don't use 'em, it hardly matters... */ -NON_EXEC_STATIC void -PgArchiverMain(int argc, char *argv[]) +void +PgArchiverMain(pg_attribute_unused() int argc, pg_attribute_unused() char *arg[]) { + /* Drop our connection to postmaster's shared memory, as well */ + dsm_detach_all(); + PGSharedMemoryDetach(); + /* * Ignore all signals usually bound to some action in the postmaster, * except for SIGHUP, SIGTERM, SIGUSR1, SIGUSR2, and SIGQUIT. diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 462b4d7e06..27af4fc1ed 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -275,11 +275,7 @@ static instr_time total_func_time; * Local function forward declarations * ---------- */ -#ifdef EXEC_BACKEND -static pid_t pgstat_forkexec(void); -#endif - -NON_EXEC_STATIC void PgstatCollectorMain(int argc, char *argv[]) pg_attribute_noreturn(); +void PgstatCollectorMain(int argc, char *argv[]) pg_attribute_noreturn(); static void pgstat_beshutdown_hook(int code, Datum arg); static PgStat_StatDBEntry *pgstat_get_db_entry(Oid databaseid, bool create); @@ -685,46 +681,18 @@ pgstat_reset_all(void) pgstat_reset_remove_files(PGSTAT_STAT_PERMANENT_DIRECTORY); } -#ifdef EXEC_BACKEND - -/* - * pgstat_forkexec() - - * - * Format up the arglist for, then fork and exec, statistics collector process - */ -static pid_t -pgstat_forkexec(void) -{ - char *av[10]; - int ac = 0; - - av[ac++] = "postgres"; - av[ac++] = "--forkcol"; - av[ac++] = NULL; /* filled in by postmaster_forkexec */ - - av[ac] = NULL; - Assert(ac < lengthof(av)); - - return postmaster_forkexec(ac, av); -} -#endif /* EXEC_BACKEND */ - /* - * pgstat_start() - + * PgstatCollectorPrep * * Called from postmaster at startup or after an existing collector * died. Attempt to fire up a fresh statistics collector. * - * Returns PID of child process, or 0 if fail. - * - * Note: if fail, we will be called again from the postmaster main loop. */ int -pgstat_start(void) +PgstatCollectorPrep(int argc, char *argv[]) { time_t curtime; - pid_t pgStatPid; /* * Check that the socket is there, else pgstat_init failed and we can do @@ -733,53 +701,18 @@ pgstat_start(void) if (pgStatSock == PGINVALID_SOCKET) return 0; + curtime = time(NULL); + /* * Do nothing if too soon since last collector start. This is a safety * valve to protect against continuous respawn attempts if the collector * is dying immediately at launch. Note that since we will be re-called * from the postmaster main loop, we will get another chance later. */ - curtime = time(NULL); - if ((unsigned int) (curtime - last_pgstat_start_time) < + if ((unsigned int) (curtime - last_pgstat_start_time) >= (unsigned int) PGSTAT_RESTART_INTERVAL) - return 0; - last_pgstat_start_time = curtime; + last_pgstat_start_time = curtime; - /* - * Okay, fork off the collector. - */ -#ifdef EXEC_BACKEND - switch ((pgStatPid = pgstat_forkexec())) -#else - switch ((pgStatPid = fork_process())) -#endif - { - case -1: - ereport(LOG, - (errmsg("could not fork statistics collector: %m"))); - return 0; - -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - - /* Drop our connection to postmaster's shared memory, as well */ - dsm_detach_all(); - PGSharedMemoryDetach(); - - PgstatCollectorMain(0, NULL); - break; -#endif - - default: - return (int) pgStatPid; - } - - /* shouldn't get here */ return 0; } @@ -4423,13 +4356,17 @@ pgstat_send_bgwriter(void) * The argc/argv parameters are valid only in EXEC_BACKEND case. * ---------- */ -NON_EXEC_STATIC void -PgstatCollectorMain(int argc, char *argv[]) +void +PgstatCollectorMain(pg_attribute_unused() int argc, pg_attribute_unused() char *argv[]) { int len; PgStat_Msg msg; int wr; + /* Drop our connection to postmaster's shared memory, as well */ + dsm_detach_all(); + PGSharedMemoryDetach(); + /* * Ignore all signals usually bound to some action in the postmaster, * except SIGHUP and SIGQUIT. Note we don't need a SIGUSR1 handler to diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 55187eb910..45723c1cd3 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -114,6 +114,7 @@ #include "postmaster/fork_process.h" #include "postmaster/pgarch.h" #include "postmaster/postmaster.h" +#include "postmaster/subprocess.h" #include "postmaster/syslogger.h" #include "replication/logicallauncher.h" #include "replication/walsender.h" @@ -136,62 +137,16 @@ #include "storage/spin.h" #endif - -/* - * Possible types of a backend. Beyond being the possible bkend_type values in - * struct bkend, these are OR-able request flag bits for SignalSomeChildren() - * and CountChildren(). - */ -#define BACKEND_TYPE_NORMAL 0x0001 /* normal backend */ -#define BACKEND_TYPE_AUTOVAC 0x0002 /* autovacuum worker process */ -#define BACKEND_TYPE_WALSND 0x0004 /* walsender process */ -#define BACKEND_TYPE_BGWORKER 0x0008 /* bgworker process */ -#define BACKEND_TYPE_ALL 0x000F /* OR of all the above */ - -#define BACKEND_TYPE_WORKER (BACKEND_TYPE_AUTOVAC | BACKEND_TYPE_BGWORKER) - -/* - * List of active backends (or child processes anyway; we don't actually - * know whether a given child has become a backend or is still in the - * authorization phase). This is used mainly to keep track of how many - * children we have and send them appropriate signals when necessary. - * - * "Special" children such as the startup, bgwriter and autovacuum launcher - * tasks are not in this list. Autovacuum worker and walsender are in it. - * Also, "dead_end" children are in it: these are children launched just for - * the purpose of sending a friendly rejection message to a would-be client. - * We must track them because they are attached to shared memory, but we know - * they will never become live backends. dead_end children are not assigned a - * PMChildSlot. - * - * Background workers are in this list, too. - */ -typedef struct bkend -{ - pid_t pid; /* process id of backend */ - int32 cancel_key; /* cancel key for cancels for this backend */ - int child_slot; /* PMChildSlot for this backend, if any */ - - /* - * Flavor of backend or auxiliary process. Note that BACKEND_TYPE_WALSND - * backends initially announce themselves as BACKEND_TYPE_NORMAL, so if - * bkend_type is normal, you should check for a recent transition. - */ - int bkend_type; - bool dead_end; /* is it going to send an error and quit? */ - bool bgworker_notify; /* gets bgworker start/stop notifications */ - dlist_node elem; /* list link in BackendList */ -} Backend; - -static dlist_head BackendList = DLIST_STATIC_INIT(BackendList); +dlist_head BackendList = DLIST_STATIC_INIT(BackendList); #ifdef EXEC_BACKEND static Backend *ShmemBackendArray; #endif BackgroundWorker *MyBgworkerEntry = NULL; - - +RegisteredBgWorker *CurrentBgWorker = NULL; +static Backend *MyBackend; +static int child_errno; /* The socket number we are listening for connections on */ int PostPortNumber; @@ -403,25 +358,25 @@ static void BackendInitialize(Port *port); static void BackendRun(Port *port) pg_attribute_noreturn(); static void ExitPostmaster(int status) pg_attribute_noreturn(); static int ServerLoop(void); -static int BackendStartup(Port *port); static int ProcessStartupPacket(Port *port, bool secure_done); static void SendNegotiateProtocolVersion(List *unrecognized_protocol_options); static void processCancelRequest(Port *port, void *pkt); static int initMasks(fd_set *rmask); static void report_fork_failure_to_client(Port *port, int errnum); static CAC_state canAcceptConnections(int backend_type); -static bool RandomCancelKey(int32 *cancel_key); static void signal_child(pid_t pid, int signal); static bool SignalSomeChildren(int signal, int targets); static void TerminateChildren(int signal); +#ifdef EXEC_BACKEND +static void shmemSetup(bool aux_process); +#endif #define SignalChildren(sig) SignalSomeChildren(sig, BACKEND_TYPE_ALL) static int CountChildren(int target); -static bool assign_backendlist_entry(RegisteredBgWorker *rw); static void maybe_start_bgworkers(void); static bool CreateOptsFile(int argc, char *argv[], char *fullprogname); -static pid_t StartChildProcess(AuxProcType type); +static pid_t StartSubprocess(SubprocessType type); static void StartAutovacuumWorker(void); static void MaybeStartWalReceiver(void); static void InitPostmasterDeathWatchHandle(void); @@ -455,8 +410,7 @@ typedef struct } win32_deadchild_waitinfo; #endif /* WIN32 */ -static pid_t backend_forkexec(Port *port); -static pid_t internal_forkexec(int argc, char *argv[], Port *port); +static pid_t internal_forkexec(int argc, char *argv[]); /* Type for a socket that can be inherited to a client process */ #ifdef WIN32 @@ -507,6 +461,7 @@ typedef struct TimestampTz PgStartTime; TimestampTz PgReloadTime; pg_time_t first_syslogger_file_time; + bool redirection_done; bool IsBinaryUpgrade; int max_safe_fds; @@ -534,15 +489,23 @@ static bool save_backend_variables(BackendParameters *param, Port *port, HANDLE childProcess, pid_t childPid); #endif -static void ShmemBackendArrayAdd(Backend *bn); static void ShmemBackendArrayRemove(Backend *bn); #endif /* EXEC_BACKEND */ -#define StartupDataBase() StartChildProcess(StartupProcess) -#define StartBackgroundWriter() StartChildProcess(BgWriterProcess) -#define StartCheckpointer() StartChildProcess(CheckpointerProcess) -#define StartWalWriter() StartChildProcess(WalWriterProcess) -#define StartWalReceiver() StartChildProcess(WalReceiverProcess) +#define BGWORKER_LEN 15 + +#define StartupDataBase() StartSubprocess(StartupType) +#define StartBackgroundWriter() StartSubprocess(BgWriterType) +#define StartCheckpointer() StartSubprocess(CheckpointerType) +#define StartWalWriter() StartSubprocess(WalWriterType) +#define StartWalReceiver() StartSubprocess(WalReceiverType) +#define StartAutoVacLauncher() StartSubprocess(AutoVacLauncherType) +#define StartAutoVacWorker() StartSubprocess(AutoVacWorkerType) +#define pgstat_start() StartSubprocess(PgstatCollectorType) +#define pgarch_start() StartSubprocess(PgArchiverType) +#define SysLogger_Start() StartSubprocess(SysLoggerType) +#define do_start_bgworker() StartSubprocess(BgWorkerType) +#define BackendStartup() StartSubprocess(ClientBackendType) /* Macros to check exit status of a child process */ #define EXIT_STATUS_0(st) ((st) == 0) @@ -560,6 +523,133 @@ int postmaster_alive_fds[2] = {-1, -1}; HANDLE PostmasterHandle; #endif +static Port *ConnProcPort = NULL; + +/* + * BackendMain + * + * Child code when forking a Backend. + */ +void BackendMain(pg_attribute_unused() int argc, pg_attribute_unused() char *argv[]) +{ + /* + * Perform additional initialization and collect startup packet. + * + * We want to do this before InitProcess() for a couple of reasons: 1. + * so that we aren't eating up a PGPROC slot while waiting on the + * client. 2. so that if InitProcess() fails due to being out of + * PGPROC slots, we have already initialized libpq and are able to + * report the error to the client. + */ + BackendInitialize(ConnProcPort); + +#ifdef EXEC_BACKEND + shmemSetup(false); +#endif + + /* And run the backend */ + BackendRun(ConnProcPort); /* does not return */ +} + +/* + * BackendPostmasterMain + * + * Parent code when forking a Backend. + */ +void BackendParentMain(pg_attribute_unused() int argc, pg_attribute_unused() char *argv[]) +{ + /* in parent, successful fork */ + ereport(DEBUG2, + (errmsg_internal("forked new backend, pid=%d socket=%d", + (int) MyChildProcPid, (int) MyProcPort->sock))); + + /* + * Everything's been successful, it's safe to add this backend to our list + * of backends. + */ + MyBackend->pid = MyChildProcPid; + MyBackend->bkend_type = BACKEND_TYPE_NORMAL; /* Can change later to WALSND */ + dlist_push_head(&BackendList, &MyBackend->elem); + +#ifdef EXEC_BACKEND + if (!MyBackend->dead_end) + ShmemBackendArrayAdd(MyBackend); +#endif +} + +/* + * BackendCleanup + * + * Backend cleanup in case a failure occurs forking a new Backend. + */ +bool +BackendCleanup(int argc, char *argv[]) +{ + if (!MyBackend->dead_end) + (void) ReleasePostmasterChildSlot(MyBackend->child_slot); + free(MyBackend); + + report_fork_failure_to_client(MyProcPort, child_errno); + + /* Don't panic */ + return false; +} + +/* + * BackendPrep + * + * Prepare a ForkProcType struct for starting a Backend. + * This does all prep related to av parameters and error messages. +*/ +int +BackendPrep(int argc, char *argv[]) +{ + /* + * Create backend data structure. Better before the fork() so we can + * handle failure cleanly. + */ + MyBackend = (Backend *) malloc(sizeof(Backend)); + if (!MyBackend) + { + ereport(LOG, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); + } + + /* + * Compute the cancel key that will be assigned to this backend. The + * backend will have its own copy in the forked-off process' value of + * MyCancelKey, so that it can transmit the key to the frontend. + */ + if (!RandomCancelKey(&MyCancelKey)) + { + free(MyBackend); + ereport(LOG, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random cancel key"))); + } + + MyBackend->cancel_key = MyCancelKey; + + /* Pass down canAcceptConnections state */ + ConnProcPort->canAcceptConnections = canAcceptConnections(BACKEND_TYPE_NORMAL); + MyBackend->dead_end = (ConnProcPort->canAcceptConnections != CAC_OK && + ConnProcPort->canAcceptConnections != CAC_WAITBACKUP); + + /* + * Unless it's a dead_end child, assign it a child slot number + */ + if (!MyBackend->dead_end) + MyBackend->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); + else + MyBackend->child_slot = 0; + + /* Hasn't asked to be notified about any bgworkers yet */ + MyBackend->bgworker_notify = false; + + return 0; +} + /* * Postmaster main entry point */ @@ -1719,19 +1809,17 @@ ServerLoop(void) break; if (FD_ISSET(ListenSocket[i], &rmask)) { - Port *port; - - port = ConnCreate(ListenSocket[i]); - if (port) + ConnProcPort = ConnCreate(ListenSocket[i]); + if (ConnProcPort) { - BackendStartup(port); + BackendStartup(); /* * We no longer need the open socket or port structure * in this process */ - StreamClose(port->sock); - ConnFree(port); + StreamClose(ConnProcPort->sock); + ConnFree(ConnProcPort); } } } @@ -2533,12 +2621,9 @@ ConnFree(Port *conn) * This is called during child process startup to release file descriptors * that are not needed by that child process. The postmaster still has * them open, of course. - * - * Note: we pass am_syslogger as a boolean because we don't want to set - * the global variable yet when this is called. */ void -ClosePostmasterPorts(bool am_syslogger) +ClosePostmasterPorts(void) { int i; @@ -2575,7 +2660,8 @@ ClosePostmasterPorts(bool am_syslogger) * If using syslogger, close the read side of the pipe. We don't bother * tracking this in fd.c, either. */ - if (!am_syslogger) + if (strcmp(MySubprocess->name, "logger") || + strcmp(MySubprocess->name, "backend")) { #ifndef WIN32 if (syslogPipe[0] >= 0) @@ -4118,122 +4204,6 @@ TerminateChildren(int signal) signal_child(PgStatPID, signal); } -/* - * BackendStartup -- start backend process - * - * returns: STATUS_ERROR if the fork failed, STATUS_OK otherwise. - * - * Note: if you change this code, also consider StartAutovacuumWorker. - */ -static int -BackendStartup(Port *port) -{ - Backend *bn; /* for backend cleanup */ - pid_t pid; - - /* - * Create backend data structure. Better before the fork() so we can - * handle failure cleanly. - */ - bn = (Backend *) malloc(sizeof(Backend)); - if (!bn) - { - ereport(LOG, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); - return STATUS_ERROR; - } - - /* - * Compute the cancel key that will be assigned to this backend. The - * backend will have its own copy in the forked-off process' value of - * MyCancelKey, so that it can transmit the key to the frontend. - */ - if (!RandomCancelKey(&MyCancelKey)) - { - free(bn); - ereport(LOG, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate random cancel key"))); - return STATUS_ERROR; - } - - bn->cancel_key = MyCancelKey; - - /* Pass down canAcceptConnections state */ - port->canAcceptConnections = canAcceptConnections(BACKEND_TYPE_NORMAL); - bn->dead_end = (port->canAcceptConnections != CAC_OK && - port->canAcceptConnections != CAC_WAITBACKUP); - - /* - * Unless it's a dead_end child, assign it a child slot number - */ - if (!bn->dead_end) - bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); - else - bn->child_slot = 0; - - /* Hasn't asked to be notified about any bgworkers yet */ - bn->bgworker_notify = false; - -#ifdef EXEC_BACKEND - pid = backend_forkexec(port); -#else /* !EXEC_BACKEND */ - pid = fork_process(); - if (pid == 0) /* child */ - { - free(bn); - - /* Detangle from postmaster */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - - /* Perform additional initialization and collect startup packet */ - BackendInitialize(port); - - /* And run the backend */ - BackendRun(port); - } -#endif /* EXEC_BACKEND */ - - if (pid < 0) - { - /* in parent, fork failed */ - int save_errno = errno; - - if (!bn->dead_end) - (void) ReleasePostmasterChildSlot(bn->child_slot); - free(bn); - errno = save_errno; - ereport(LOG, - (errmsg("could not fork new process for connection: %m"))); - report_fork_failure_to_client(port, save_errno); - return STATUS_ERROR; - } - - /* in parent, successful fork */ - ereport(DEBUG2, - (errmsg_internal("forked new backend, pid=%d socket=%d", - (int) pid, (int) port->sock))); - - /* - * Everything's been successful, it's safe to add this backend to our list - * of backends. - */ - bn->pid = pid; - bn->bkend_type = BACKEND_TYPE_NORMAL; /* Can change later to WALSND */ - dlist_push_head(&BackendList, &bn->elem); - -#ifdef EXEC_BACKEND - if (!bn->dead_end) - ShmemBackendArrayAdd(bn); -#endif - - return STATUS_OK; -} - /* * Try to report backend fork() failure to client before we close the * connection. Since we do not care to risk blocking the postmaster on @@ -4476,7 +4446,7 @@ BackendRun(Port *port) maxac * sizeof(char *)); ac = 0; - av[ac++] = "postgres"; + av[ac++] = pstrdup("postgres"); /* * Pass any backend switches specified with -o on the postmaster's own @@ -4527,36 +4497,7 @@ BackendRun(Port *port) pid_t postmaster_forkexec(int argc, char *argv[]) { - Port port; - - /* This entry point passes dummy values for the Port variables */ - memset(&port, 0, sizeof(port)); - return internal_forkexec(argc, argv, &port); -} - -/* - * backend_forkexec -- fork/exec off a backend process - * - * Some operating systems (WIN32) don't have fork() so we have to simulate - * it by storing parameters that need to be passed to the child and - * then create a new child process. - * - * returns the pid of the fork/exec'd process, or -1 on failure - */ -static pid_t -backend_forkexec(Port *port) -{ - char *av[4]; - int ac = 0; - - av[ac++] = "postgres"; - av[ac++] = "--forkbackend"; - av[ac++] = NULL; /* filled in by internal_forkexec */ - - av[ac] = NULL; - Assert(ac < lengthof(av)); - - return internal_forkexec(ac, av, port); + return internal_forkexec(argc, argv); } #ifndef WIN32 @@ -4568,7 +4509,7 @@ backend_forkexec(Port *port) * - fork():s, and then exec():s the child process */ static pid_t -internal_forkexec(int argc, char *argv[], Port *port) +internal_forkexec(int argc, char *argv[]) { static unsigned long tmpBackendFileNum = 0; pid_t pid; @@ -4576,7 +4517,7 @@ internal_forkexec(int argc, char *argv[], Port *port) BackendParameters param; FILE *fp; - if (!save_backend_variables(¶m, port)) + if (!save_backend_variables(¶m, ConnProcPort)) return -1; /* log made by save_backend_variables */ /* Calculate name for temp file */ @@ -4861,6 +4802,29 @@ retry: } #endif /* WIN32 */ +/* + * shmemSetup + * + * Helper function for a child to set up shmem before + * executing. + * + * aux_process - set to true if an auxiliary process. + */ +static void +shmemSetup(bool aux_process) +{ + /* Restore basic shared memory pointers */ + InitShmemAccess(UsedShmemSegAddr); + + /* Need a PGPROC to run CreateSharedMemoryAndSemaphores */ + if (aux_process) + InitAuxiliaryProcess(); + else + InitProcess(); + + /* Attach process to shared data structures */ + CreateSharedMemoryAndSemaphores(); +} /* * SubPostmasterMain -- Get the fork/exec'd process into a state equivalent @@ -4892,12 +4856,15 @@ SubPostmasterMain(int argc, char *argv[]) if (argc < 3) elog(FATAL, "invalid subpostmaster invocation"); + Assert(!ConnProcPort); + /* Read in the variables file */ memset(&port, 0, sizeof(Port)); read_backend_variables(argv[2], &port); + ConnProcPort = &port; /* Close the postmaster's sockets (as soon as we know them) */ - ClosePostmasterPorts(strcmp(argv[1], "--forklog") == 0); + ClosePostmasterPorts(); /* * Set reference point for stack-depth checking @@ -4936,16 +4903,16 @@ SubPostmasterMain(int argc, char *argv[]) strcmp(argv[1], "--forkavlauncher") == 0 || strcmp(argv[1], "--forkavworker") == 0 || strcmp(argv[1], "--forkboot") == 0 || - strncmp(argv[1], "--forkbgworker=", 15) == 0) + strncmp(argv[1], "--forkbgworker=", BGWORKER_LEN) == 0) PGSharedMemoryReAttach(); else PGSharedMemoryNoReAttach(); /* autovacuum needs this set before calling InitProcess */ if (strcmp(argv[1], "--forkavlauncher") == 0) - AutovacuumLauncherIAm(); + am_autovacuum_launcher = true; if (strcmp(argv[1], "--forkavworker") == 0) - AutovacuumWorkerIAm(); + am_autovacuum_worker = true; /* * Start our win32 signal implementation. This has to be done after we @@ -5038,64 +5005,33 @@ SubPostmasterMain(int argc, char *argv[]) } if (strcmp(argv[1], "--forkboot") == 0) { - /* Restore basic shared memory pointers */ - InitShmemAccess(UsedShmemSegAddr); - - /* Need a PGPROC to run CreateSharedMemoryAndSemaphores */ - InitAuxiliaryProcess(); - - /* Attach process to shared data structures */ - CreateSharedMemoryAndSemaphores(); - + shmemSetup(true); AuxiliaryProcessMain(argc - 2, argv + 2); /* does not return */ } if (strcmp(argv[1], "--forkavlauncher") == 0) { - /* Restore basic shared memory pointers */ - InitShmemAccess(UsedShmemSegAddr); - - /* Need a PGPROC to run CreateSharedMemoryAndSemaphores */ - InitProcess(); - - /* Attach process to shared data structures */ - CreateSharedMemoryAndSemaphores(); - + shmemSetup(true); AutoVacLauncherMain(argc - 2, argv + 2); /* does not return */ } if (strcmp(argv[1], "--forkavworker") == 0) { - /* Restore basic shared memory pointers */ - InitShmemAccess(UsedShmemSegAddr); - - /* Need a PGPROC to run CreateSharedMemoryAndSemaphores */ - InitProcess(); - - /* Attach process to shared data structures */ - CreateSharedMemoryAndSemaphores(); - + shmemSetup(true); AutoVacWorkerMain(argc - 2, argv + 2); /* does not return */ } - if (strncmp(argv[1], "--forkbgworker=", 15) == 0) + if (strncmp(argv[1], "--forkbgworker=", BGWORKER_LEN) == 0) { int shmem_slot; /* do this as early as possible; in particular, before InitProcess() */ IsBackgroundWorker = true; - /* Restore basic shared memory pointers */ - InitShmemAccess(UsedShmemSegAddr); - - /* Need a PGPROC to run CreateSharedMemoryAndSemaphores */ - InitProcess(); - - /* Attach process to shared data structures */ - CreateSharedMemoryAndSemaphores(); + shmemSetup(false); /* Fetch MyBgworkerEntry from shared memory */ - shmem_slot = atoi(argv[1] + 15); + shmem_slot = atoi(argv[1] + BGWORKER_LEN); MyBgworkerEntry = BackgroundWorkerEntry(shmem_slot); - StartBackgroundWorker(); + BackgroundWorkerMain(argc, argv); } if (strcmp(argv[1], "--forkarch") == 0) { @@ -5378,7 +5314,7 @@ StartupPacketTimeoutHandler(void) /* * Generate a random cancel key. */ -static bool +bool RandomCancelKey(int32 *cancel_key) { return pg_strong_random(cancel_key, sizeof(int32)); @@ -5424,7 +5360,6 @@ CountChildren(int target) return cnt; } - /* * StartChildProcess -- start an auxiliary process for the postmaster * @@ -5435,93 +5370,118 @@ CountChildren(int target) * to start subprocess. */ static pid_t -StartChildProcess(AuxProcType type) +StartSubprocess(SubprocessType type) { pid_t pid; - char *av[10]; - int ac = 0; + char *argv[10]; + int argc = 0; char typebuf[32]; - /* - * Set up command-line arguments for subprocess - */ - av[ac++] = "postgres"; + /* This should be our first time in this function */ + //Assert(!MySubprocess); + InitializeMySubprocess(type); + + argv[argc++] = "postgres"; #ifdef EXEC_BACKEND - av[ac++] = "--forkboot"; - av[ac++] = NULL; /* filled in by postmaster_forkexec */ + argv[argc++] = psprintf("--%s", MySubprocess->progname); + argv[argc++] = NULL; #endif snprintf(typebuf, sizeof(typebuf), "-x%d", type); - av[ac++] = typebuf; + argv[argc++] = typebuf; - av[ac] = NULL; - Assert(ac < lengthof(av)); + argv[argc] = NULL; + Assert(argc < lengthof(argv)); + + /* Prep the subprocesses for a fork */ + if (MySubprocess->init) + { + int rc = 0; + rc = MySubprocess->init(argc, argv); + + if (rc != 0) + { + ereport(WARNING, + (errmsg("subprocess %s returned with %d: %m", + MySubprocess->desc, rc))); + } + } #ifdef EXEC_BACKEND - pid = postmaster_forkexec(ac, av); -#else /* !EXEC_BACKEND */ + pid = postmaster_forkexec(argc, argv); +#else pid = fork_process(); +#endif + + /* some processes like backends and bgworkers need the pid */ + MyChildProcPid = pid; if (pid == 0) /* child */ { + +#ifdef EXEC_BACKEND + /* Call the process's main subroutine */ + if (MySubprocess->needs_aux_proc) + AuxiliaryProcessMain(argc, argv); + else + MySubprocess->entrypoint(argc, argv); +#else InitPostmasterChild(); + ClosePostmasterPorts(); - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); + /* + * Release postmaster's working memory context unless we're still using + * it. + */ + if (!MySubprocess->keep_postmaster_memcontext) + { + MemoryContextSwitchTo(TopMemoryContext); + MemoryContextDelete(PostmasterContext); + + PostmasterContext = NULL; + } - /* Release postmaster's working memory context */ - MemoryContextSwitchTo(TopMemoryContext); - MemoryContextDelete(PostmasterContext); - PostmasterContext = NULL; + /* Call the process's main subroutine */ + if (MySubprocess->needs_aux_proc) + AuxiliaryProcessMain(argc, argv); + else + MySubprocess->entrypoint(argc, argv); - AuxiliaryProcessMain(ac, av); ExitPostmaster(0); +#endif } -#endif /* EXEC_BACKEND */ - if (pid < 0) { /* in parent, fork failed */ - int save_errno = errno; + child_errno = errno; - errno = save_errno; - switch (type) - { - case StartupProcess: - ereport(LOG, - (errmsg("could not fork startup process: %m"))); - break; - case BgWriterProcess: - ereport(LOG, - (errmsg("could not fork background writer process: %m"))); - break; - case CheckpointerProcess: - ereport(LOG, - (errmsg("could not fork checkpointer process: %m"))); - break; - case WalWriterProcess: - ereport(LOG, - (errmsg("could not fork WAL writer process: %m"))); - break; - case WalReceiverProcess: - ereport(LOG, - (errmsg("could not fork WAL receiver process: %m"))); - break; - default: - ereport(LOG, - (errmsg("could not fork process: %m"))); - break; - } + ereport(LOG, + (errmsg("could not fork %s: %m", MySubprocess->desc))); /* * fork failure is fatal during startup, but there's no need to choke * immediately if starting other child types fails. */ - if (type == StartupProcess) - ExitPostmaster(1); + if (MySubprocess->cleanup) + { + /* + * Panic if the cleanup function tells us to. Otherwise, just bail + * with -1. + */ + if (MySubprocess->cleanup(argc, argv)) + ExitPostmaster(1); + else + return -1; + } + return 0; } + else + { + if (MySubprocess->parent_main) + MySubprocess->parent_main(argc, argv); + } /* * in parent, successful fork @@ -5756,124 +5716,6 @@ BackgroundWorkerUnblockSignals(void) PG_SETMASK(&UnBlockSig); } -#ifdef EXEC_BACKEND -static pid_t -bgworker_forkexec(int shmem_slot) -{ - char *av[10]; - int ac = 0; - char forkav[MAXPGPATH]; - - snprintf(forkav, MAXPGPATH, "--forkbgworker=%d", shmem_slot); - - av[ac++] = "postgres"; - av[ac++] = forkav; - av[ac++] = NULL; /* filled in by postmaster_forkexec */ - av[ac] = NULL; - - Assert(ac < lengthof(av)); - - return postmaster_forkexec(ac, av); -} -#endif - -/* - * Start a new bgworker. - * Starting time conditions must have been checked already. - * - * Returns true on success, false on failure. - * In either case, update the RegisteredBgWorker's state appropriately. - * - * This code is heavily based on autovacuum.c, q.v. - */ -static bool -do_start_bgworker(RegisteredBgWorker *rw) -{ - pid_t worker_pid; - - Assert(rw->rw_pid == 0); - - /* - * Allocate and assign the Backend element. Note we must do this before - * forking, so that we can handle failures (out of memory or child-process - * slots) cleanly. - * - * Treat failure as though the worker had crashed. That way, the - * postmaster will wait a bit before attempting to start it again; if we - * tried again right away, most likely we'd find ourselves hitting the - * same resource-exhaustion condition. - */ - if (!assign_backendlist_entry(rw)) - { - rw->rw_crashed_at = GetCurrentTimestamp(); - return false; - } - - ereport(DEBUG1, - (errmsg("starting background worker process \"%s\"", - rw->rw_worker.bgw_name))); - -#ifdef EXEC_BACKEND - switch ((worker_pid = bgworker_forkexec(rw->rw_shmem_slot))) -#else - switch ((worker_pid = fork_process())) -#endif - { - case -1: - /* in postmaster, fork failed ... */ - ereport(LOG, - (errmsg("could not fork worker process: %m"))); - /* undo what assign_backendlist_entry did */ - ReleasePostmasterChildSlot(rw->rw_child_slot); - rw->rw_child_slot = 0; - free(rw->rw_backend); - rw->rw_backend = NULL; - /* mark entry as crashed, so we'll try again later */ - rw->rw_crashed_at = GetCurrentTimestamp(); - break; - -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - - /* - * Before blowing away PostmasterContext, save this bgworker's - * data where it can find it. - */ - MyBgworkerEntry = (BackgroundWorker *) - MemoryContextAlloc(TopMemoryContext, sizeof(BackgroundWorker)); - memcpy(MyBgworkerEntry, &rw->rw_worker, sizeof(BackgroundWorker)); - - /* Release postmaster's working memory context */ - MemoryContextSwitchTo(TopMemoryContext); - MemoryContextDelete(PostmasterContext); - PostmasterContext = NULL; - - StartBackgroundWorker(); - - exit(1); /* should not get here */ - break; -#endif - default: - /* in postmaster, fork successful ... */ - rw->rw_pid = worker_pid; - rw->rw_backend->pid = rw->rw_pid; - ReportBackgroundWorkerPID(rw); - /* add new worker to lists of backends */ - dlist_push_head(&BackendList, &rw->rw_backend->elem); -#ifdef EXEC_BACKEND - ShmemBackendArrayAdd(rw->rw_backend); -#endif - return true; - } - - return false; -} - /* * Does the current postmaster state require starting a worker with the * specified start_time? @@ -5914,67 +5756,6 @@ bgworker_should_start_now(BgWorkerStartTime start_time) return false; } -/* - * Allocate the Backend struct for a connected background worker, but don't - * add it to the list of backends just yet. - * - * On failure, return false without changing any worker state. - * - * Some info from the Backend is copied into the passed rw. - */ -static bool -assign_backendlist_entry(RegisteredBgWorker *rw) -{ - Backend *bn; - - /* - * Check that database state allows another connection. Currently the - * only possible failure is CAC_TOOMANY, so we just log an error message - * based on that rather than checking the error code precisely. - */ - if (canAcceptConnections(BACKEND_TYPE_BGWORKER) != CAC_OK) - { - ereport(LOG, - (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), - errmsg("no slot available for new worker process"))); - return false; - } - - /* - * Compute the cancel key that will be assigned to this session. We - * probably don't need cancel keys for background workers, but we'd better - * have something random in the field to prevent unfriendly people from - * sending cancels to them. - */ - if (!RandomCancelKey(&MyCancelKey)) - { - ereport(LOG, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate random cancel key"))); - return false; - } - - bn = malloc(sizeof(Backend)); - if (bn == NULL) - { - ereport(LOG, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); - return false; - } - - bn->cancel_key = MyCancelKey; - bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); - bn->bkend_type = BACKEND_TYPE_BGWORKER; - bn->dead_end = false; - bn->bgworker_notify = false; - - rw->rw_backend = bn; - rw->rw_child_slot = bn->child_slot; - - return true; -} - /* * If the time is right, start background worker(s). * @@ -6079,7 +5860,8 @@ maybe_start_bgworkers(void) * crashed, but there's no need because the next run of this * function will do that. */ - if (!do_start_bgworker(rw)) + CurrentBgWorker = rw; + if (do_start_bgworker() <= 0) { StartWorkerNeeded = true; return; @@ -6486,7 +6268,7 @@ ShmemBackendArrayAllocation(void) memset(ShmemBackendArray, 0, size); } -static void +void ShmemBackendArrayAdd(Backend *bn) { /* The array slot corresponding to my PMChildSlot should be free */ diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index c2250d7d4e..8957b341b9 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -127,13 +127,19 @@ HandleStartupProcInterrupts(void) ProcessProcSignalBarrier(); } +bool +StartupCleanup(int argc, char *argv[]) +{ + /* Panic! */ + return true; +} /* ---------------------------------- * Startup Process main entry point * ---------------------------------- */ void -StartupProcessMain(void) +StartupProcessMain(int argc, char *argv[]) { /* * Properly accept or ignore signals the postmaster might send us. diff --git a/src/backend/postmaster/subprocess.c b/src/backend/postmaster/subprocess.c new file mode 100644 index 0000000000..acd7da9213 --- /dev/null +++ b/src/backend/postmaster/subprocess.c @@ -0,0 +1,206 @@ +/*------------------------------------------------------------------------- + * + * subprocess.c + * + * Copyright (c) 2004-2020, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/postmaster/syslogger.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "postmaster/subprocess.h" +#include "bootstrap/bootstrap.h" +#include "postmaster/bgwriter.h" +#include "postmaster/walwriter.h" +#include "postmaster/syslogger.h" +#include "postmaster/autovacuum.h" +#include "postmaster/bgworker_internals.h" +#include "postmaster/bgworker.h" +#include "postmaster/postmaster.h" +#include "postmaster/startup.h" +#include "pgstat.h" +#include "replication/walreceiver.h" + +PgSubprocess *MySubprocess = NULL; + +PgSubprocess process_types[] = { + { + .progname = "boot", + .name = "checker", + .desc = "checker process", + .needs_shmem = true, + .needs_aux_proc = true, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = CheckerModeMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "boot", + .name = "bootstrap", + .desc = "bootstrap process", + .needs_shmem = true, + .needs_aux_proc = true, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = BootstrapModeMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "boot", + .name = "startup", + .desc = "startup process", + .needs_shmem = true, + .needs_aux_proc = true, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = StartupProcessMain, + .cleanup = StartupCleanup, + .parent_main = NULL + }, + { + .progname = "boot", + .name = "bgwriter", + .desc = "background writer process", + .needs_shmem = true, + .needs_aux_proc = true, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = BackgroundWriterMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "boot", + .name = "checkpointer", + .desc = "checkpointer process", + .needs_shmem = true, + .needs_aux_proc = true, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = CheckpointerMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "boot", + .name = "walwriter", + .desc = "WAL writer process", + .needs_shmem = true, + .needs_aux_proc = true, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = WalWriterMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "boot", + .name = "walreceiver", + .desc = "WAL receiver process", + .needs_shmem = true, + .needs_aux_proc = true, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = WalReceiverMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "forkavlauncher", + .name = "autovacuum launcher", + .desc = "autovacuum launcher process", + .needs_shmem = true, + .needs_aux_proc = false, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = AutoVacLauncherMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "forkavworker", + .name = "autovacuum worker", + .desc = "autovacuum worker process", + .needs_shmem = true, + .needs_aux_proc = false, + .keep_postmaster_memcontext = false, + .init = NULL, + .entrypoint = AutoVacWorkerMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "forkcol", + .name = "pgstat collector", + .desc = "statistics collector process", + .needs_shmem = true, + .needs_aux_proc = false, + .keep_postmaster_memcontext = false, + .init = PgstatCollectorPrep, + .entrypoint = PgstatCollectorMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "forkarch", + .name = "archiver", + .desc = "archiver process", + .needs_shmem = true, + .needs_aux_proc = false, + .keep_postmaster_memcontext = false, + .init = PgArchiverPrep, + .entrypoint = PgArchiverMain, + .cleanup = NULL, + .parent_main = NULL + }, + { + .progname = "forklog", + .name = "logger", + .desc = "system logger", + .needs_shmem = true, + .needs_aux_proc = false, + .keep_postmaster_memcontext = true, + .init = SysLoggerPrep, + .entrypoint = SysLoggerMain, + .cleanup = NULL, + .parent_main = SysLoggerParentMain + }, + { + .progname = "forkbgworker", + .name = "bgworker", + .desc = "background worker", + .needs_shmem = true, + .needs_aux_proc = false, + .keep_postmaster_memcontext = false, + .init = BgWorkerPrep, + .entrypoint = BackgroundWorkerMain, + .cleanup = BackgroundWorkerCleanup, + .parent_main = BackgroundWorkerParentMain + }, + { + .progname = "forkbackend", + .name = "backend", + .desc = "client backend", + .needs_shmem = true, + .needs_aux_proc = false, + .keep_postmaster_memcontext = false, + .init = BackendPrep, + .entrypoint = BackendMain, + .cleanup = BackendCleanup, + .parent_main = BackendParentMain + } +}; + +void +InitializeMySubprocess(SubprocessType type) +{ + /* Globally set subprocess information */ + MySubprocess = (PgSubprocess *)malloc(sizeof(PgSubprocess)); + memcpy(MySubprocess, &process_types[type], sizeof(PgSubprocess)); +} diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index cf7b535e4e..6cb35c9165 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -48,6 +48,7 @@ #include "storage/pg_shmem.h" #include "tcop/tcopprot.h" #include "utils/guc.h" +#include "utils/memutils.h" #include "utils/ps_status.h" #include "utils/timestamp.h" @@ -93,6 +94,7 @@ NON_EXEC_STATIC pg_time_t first_syslogger_file_time = 0; static char *last_file_name = NULL; static char *last_csv_file_name = NULL; + /* * Buffers for saving partial messages from different backends. * @@ -131,13 +133,15 @@ static CRITICAL_SECTION sysloggerSection; static volatile sig_atomic_t got_SIGHUP = false; static volatile sig_atomic_t rotation_requested = false; - /* Local subroutines */ #ifdef EXEC_BACKEND -static pid_t syslogger_forkexec(void); static void syslogger_parseArgs(int argc, char *argv[]); #endif -NON_EXEC_STATIC void SysLoggerMain(int argc, char *argv[]) pg_attribute_noreturn(); + +/* Subprocess start routines */ +void SysLoggerMain(int argc, char *argv[]) pg_attribute_noreturn(); +void SysLoggerPostmasterMain(int argc, char *argv[]); + static void process_pipe_input(char *logbuffer, int *bytes_in_logbuffer); static void flush_pipe_input(char *logbuffer, int *bytes_in_logbuffer); static FILE *logfile_open(const char *filename, const char *mode, @@ -158,7 +162,7 @@ static void update_metainfo_datafile(void); * Main entry point for syslogger process * argc/argv parameters are valid only in EXEC_BACKEND case. */ -NON_EXEC_STATIC void +void SysLoggerMain(int argc, char *argv[]) { #ifndef WIN32 @@ -171,12 +175,16 @@ SysLoggerMain(int argc, char *argv[]) pg_time_t now; WaitEventSet *wes; - now = MyStartTime; - #ifdef EXEC_BACKEND syslogger_parseArgs(argc, argv); +#else + /* Drop our connection to postmaster's shared memory, as well */ + dsm_detach_all(); + PGSharedMemoryDetach(); #endif /* EXEC_BACKEND */ + now = MyStartTime; + am_syslogger = true; init_ps_display("logger", "", "", ""); @@ -540,17 +548,47 @@ SysLoggerMain(int argc, char *argv[]) } /* - * Postmaster subroutine to start a syslogger subprocess. + * Helper function for setting up a file number buffer + */ +static char * +setup_file_buff(FILE *file) +{ + char *filenobuf; + + /* static variables (those not passed by write_backend_variables) */ +#ifndef WIN32 + if (file != NULL) + filenobuf = psprintf("%d", fileno(file)); + else + filenobuf = pstrdup("-1"); +#else /* WIN32 */ + if (file != NULL) + filenobuf = psprintf("%ld", (long) _get_osfhandle(_fileno(file))); + else + filenobuf = pstrdup("0"); +#endif /* WIN32 */ + + return filenobuf; +} + +/* + * SysLoggerPrep + * + * Postmaster subroutine to prepare a syslogger subprocess. */ int -SysLogger_Start(void) +SysLoggerPrep(int argc, char *argv[]) { - pid_t sysloggerPid; char *filename; + MemoryContext cxt; if (!Logging_collector) return 0; + cxt = MemoryContextSwitchTo(PostmasterContext); + + elog(LOG, "setting up the syslogger"); + /* * If first time through, create the pipe which will receive stderr * output. @@ -631,167 +669,94 @@ SysLogger_Start(void) pfree(filename); } -#ifdef EXEC_BACKEND - switch ((sysloggerPid = syslogger_forkexec())) -#else - switch ((sysloggerPid = fork_process())) -#endif - { - case -1: - ereport(LOG, - (errmsg("could not fork system logger: %m"))); - return 0; - -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(true); - - /* Drop our connection to postmaster's shared memory, as well */ - dsm_detach_all(); - PGSharedMemoryDetach(); - - /* do the work */ - SysLoggerMain(0, NULL); - break; -#endif - - default: - /* success, in postmaster */ - - /* now we redirect stderr, if not done already */ - if (!redirection_done) - { -#ifdef WIN32 - int fd; -#endif - - /* - * Leave a breadcrumb trail when redirecting, in case the user - * forgets that redirection is active and looks only at the - * original stderr target file. - */ - ereport(LOG, - (errmsg("redirecting log output to logging collector process"), - errhint("Future log output will appear in directory \"%s\".", - Log_directory))); + argv[argc++] = setup_file_buff(syslogFile); + argv[argc++] = setup_file_buff(csvlogFile); + argv[argc] = NULL; -#ifndef WIN32 - fflush(stdout); - if (dup2(syslogPipe[1], fileno(stdout)) < 0) - ereport(FATAL, - (errcode_for_file_access(), - errmsg("could not redirect stdout: %m"))); - fflush(stderr); - if (dup2(syslogPipe[1], fileno(stderr)) < 0) - ereport(FATAL, - (errcode_for_file_access(), - errmsg("could not redirect stderr: %m"))); - /* Now we are done with the write end of the pipe. */ - close(syslogPipe[1]); - syslogPipe[1] = -1; -#else - - /* - * open the pipe in binary mode and make sure stderr is binary - * after it's been dup'ed into, to avoid disturbing the pipe - * chunking protocol. - */ - fflush(stderr); - fd = _open_osfhandle((intptr_t) syslogPipe[1], - _O_APPEND | _O_BINARY); - if (dup2(fd, _fileno(stderr)) < 0) - ereport(FATAL, - (errcode_for_file_access(), - errmsg("could not redirect stderr: %m"))); - close(fd); - _setmode(_fileno(stderr), _O_BINARY); - - /* - * Now we are done with the write end of the pipe. - * CloseHandle() must not be called because the preceding - * close() closes the underlying handle. - */ - syslogPipe[1] = 0; -#endif - redirection_done = true; - } + Assert(argc < lengthof(argv)); - /* postmaster will never write the file(s); close 'em */ - fclose(syslogFile); - syslogFile = NULL; - if (csvlogFile != NULL) - { - fclose(csvlogFile); - csvlogFile = NULL; - } - return (int) sysloggerPid; - } + MemoryContextSwitchTo(cxt); - /* we should never reach here */ return 0; } - -#ifdef EXEC_BACKEND - /* - * syslogger_forkexec() - + * SysLoggerParentMain * - * Format up the arglist for, then fork and exec, a syslogger process + * Fallthrough subroutine for parent running syslogger. */ -static pid_t -syslogger_forkexec(void) +void +SysLoggerParentMain(int argc, char *argv[]) { - char *av[10]; - int ac = 0; - char filenobuf[32]; - char csvfilenobuf[32]; + /* now we redirect stderr, if not done already */ + if (!redirection_done) + { +#ifdef WIN32 + int fd; +#endif - av[ac++] = "postgres"; - av[ac++] = "--forklog"; - av[ac++] = NULL; /* filled in by postmaster_forkexec */ + /* + * Leave a breadcrumb trail when redirecting, in case the user + * forgets that redirection is active and looks only at the + * original stderr target file. + */ + ereport(LOG, + (errmsg("redirecting log output to logging collector process"), + errhint("Future log output will appear in directory \"%s\".", + Log_directory))); - /* static variables (those not passed by write_backend_variables) */ #ifndef WIN32 - if (syslogFile != NULL) - snprintf(filenobuf, sizeof(filenobuf), "%d", - fileno(syslogFile)); - else - strcpy(filenobuf, "-1"); -#else /* WIN32 */ - if (syslogFile != NULL) - snprintf(filenobuf, sizeof(filenobuf), "%ld", - (long) _get_osfhandle(_fileno(syslogFile))); - else - strcpy(filenobuf, "0"); -#endif /* WIN32 */ - av[ac++] = filenobuf; + fflush(stdout); + if (dup2(syslogPipe[1], fileno(stdout)) < 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not redirect stdout: %m"))); + fflush(stderr); + if (dup2(syslogPipe[1], fileno(stderr)) < 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not redirect stderr: %m"))); + /* Now we are done with the write end of the pipe. */ + close(syslogPipe[1]); + syslogPipe[1] = -1; +#else -#ifndef WIN32 - if (csvlogFile != NULL) - snprintf(csvfilenobuf, sizeof(csvfilenobuf), "%d", - fileno(csvlogFile)); - else - strcpy(csvfilenobuf, "-1"); -#else /* WIN32 */ - if (csvlogFile != NULL) - snprintf(csvfilenobuf, sizeof(csvfilenobuf), "%ld", - (long) _get_osfhandle(_fileno(csvlogFile))); - else - strcpy(csvfilenobuf, "0"); -#endif /* WIN32 */ - av[ac++] = csvfilenobuf; + /* + * open the pipe in binary mode and make sure stderr is binary + * after it's been dup'ed into, to avoid disturbing the pipe + * chunking protocol. + */ + fflush(stderr); + fd = _open_osfhandle((intptr_t) syslogPipe[1], + _O_APPEND | _O_BINARY); + if (dup2(fd, _fileno(stderr)) < 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not redirect stderr: %m"))); + close(fd); + _setmode(_fileno(stderr), _O_BINARY); - av[ac] = NULL; - Assert(ac < lengthof(av)); + /* + * Now we are done with the write end of the pipe. + * CloseHandle() must not be called because the preceding + * close() closes the underlying handle. + */ + syslogPipe[1] = 0; +#endif + redirection_done = true; + } - return postmaster_forkexec(ac, av); + /* postmaster will never write the file(s); close 'em */ + fclose(syslogFile); + syslogFile = NULL; + if (csvlogFile != NULL) + { + fclose(csvlogFile); + csvlogFile = NULL; + } } +#ifdef EXEC_BACKEND /* * syslogger_parseArgs() - * @@ -800,7 +765,7 @@ syslogger_forkexec(void) static void syslogger_parseArgs(int argc, char *argv[]) { - int fd; + int fd; Assert(argc == 5); argv += 3; @@ -825,7 +790,7 @@ syslogger_parseArgs(int argc, char *argv[]) csvlogFile = fdopen(fd, "a"); setvbuf(csvlogFile, NULL, PG_IOLBF, 0); } -#else /* WIN32 */ +#else /* WIN32 */ fd = atoi(*argv++); if (fd != 0) { @@ -846,10 +811,9 @@ syslogger_parseArgs(int argc, char *argv[]) setvbuf(csvlogFile, NULL, PG_IOLBF, 0); } } -#endif /* WIN32 */ +#endif /* WIN32 */ } -#endif /* EXEC_BACKEND */ - +#endif /* -------------------------------- * pipe protocol handling diff --git a/src/backend/postmaster/walwriter.c b/src/backend/postmaster/walwriter.c index 45a2757969..3a0cec3371 100644 --- a/src/backend/postmaster/walwriter.c +++ b/src/backend/postmaster/walwriter.c @@ -85,7 +85,7 @@ int WalWriterFlushAfter = 128; * basic execution environment, but not enabled signals yet. */ void -WalWriterMain(void) +WalWriterMain(int argc, char *argv[]) { sigjmp_buf local_sigjmp_buf; MemoryContext walwriter_context; diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 2ab15c3cbb..725023f0f6 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -164,7 +164,7 @@ ProcessWalRcvInterrupts(void) /* Main entry point for walreceiver process */ void -WalReceiverMain(void) +WalReceiverMain(int argc, char *argv[]) { char conninfo[MAXCONNINFO]; char *tmp_conninfo; diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index eb19644419..3e7be0bfcd 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -38,9 +38,10 @@ volatile uint32 QueryCancelHoldoffCount = 0; volatile uint32 CritSectionCount = 0; int MyProcPid; +int MyChildProcPid; pg_time_t MyStartTime; TimestampTz MyStartTimestamp; -struct Port *MyProcPort; +struct Port *MyProcPort = NULL; int32 MyCancelKey; int MyPMChildSlot; diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 8a47dcdcb1..91e43382c0 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -1112,7 +1112,7 @@ process_startup_options(Port *port, bool am_superuser) av = (char **) palloc(maxac * sizeof(char *)); ac = 0; - av[ac++] = "postgres"; + av[ac++] = pstrdup("postgres"); pg_split_opts(av, &ac, port->cmdline_options); diff --git a/src/include/bootstrap/bootstrap.h b/src/include/bootstrap/bootstrap.h index b67fa9acc1..5d851e0081 100644 --- a/src/include/bootstrap/bootstrap.h +++ b/src/include/bootstrap/bootstrap.h @@ -15,7 +15,7 @@ #define BOOTSTRAP_H #include "nodes/execnodes.h" - +#include "postmaster/fork_process.h" /* * MAXATTR is the maximum number of attributes in a relation supported @@ -31,6 +31,8 @@ extern Relation boot_reldesc; extern Form_pg_attribute attrtypes[MAXATTR]; extern int numattr; +extern void CheckerModeMain(int argc, char *argv[]); +extern void BootstrapModeMain(int argc, char *argv[]); extern void AuxiliaryProcessMain(int argc, char *argv[]) pg_attribute_noreturn(); diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index f985453ec3..823f5258d8 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -163,6 +163,7 @@ extern PGDLLIMPORT int max_worker_processes; extern PGDLLIMPORT int max_parallel_workers; extern PGDLLIMPORT int MyProcPid; +extern PGDLLIMPORT int MyChildProcPid; extern PGDLLIMPORT pg_time_t MyStartTime; extern PGDLLIMPORT TimestampTz MyStartTimestamp; extern PGDLLIMPORT struct Port *MyProcPort; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 3a65a51696..901b709fb8 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -1237,13 +1237,12 @@ extern Size BackendStatusShmemSize(void); extern void CreateSharedBackendStatus(void); extern void pgstat_init(void); -extern int pgstat_start(void); extern void pgstat_reset_all(void); extern void allow_immediate_pgstat_restart(void); -#ifdef EXEC_BACKEND +/* Stat collector subprocess startup functions */ +extern int PgstatCollectorPrep(int argc, char *argv[]); extern void PgstatCollectorMain(int argc, char *argv[]) pg_attribute_noreturn(); -#endif /* ---------- diff --git a/src/include/postmaster/autovacuum.h b/src/include/postmaster/autovacuum.h index d40ed55531..dc02de745e 100644 --- a/src/include/postmaster/autovacuum.h +++ b/src/include/postmaster/autovacuum.h @@ -45,6 +45,10 @@ extern int AutovacuumLauncherPid; extern int Log_autovacuum_min_duration; +/* autovacuum identification */ +extern bool am_autovacuum_launcher; +extern bool am_autovacuum_worker; + /* Status inquiry functions */ extern bool AutoVacuumingActive(void); extern bool IsAutoVacuumLauncherProcess(void); @@ -58,18 +62,15 @@ extern void autovac_init(void); extern int StartAutoVacLauncher(void); extern int StartAutoVacWorker(void); +extern void AutoVacWorkerMain(int argc, char *argv[]); +extern void AutoVacLauncherMain(int argc, char *argv[]); + /* called from postmaster when a worker could not be forked */ extern void AutoVacWorkerFailed(void); /* autovacuum cost-delay balancer */ extern void AutoVacuumUpdateDelay(void); -#ifdef EXEC_BACKEND -extern void AutoVacLauncherMain(int argc, char *argv[]) pg_attribute_noreturn(); -extern void AutoVacWorkerMain(int argc, char *argv[]) pg_attribute_noreturn(); -extern void AutovacuumWorkerIAm(void); -extern void AutovacuumLauncherIAm(void); -#endif extern bool AutoVacuumRequestWork(AutoVacuumWorkItemType type, Oid relationId, BlockNumber blkno); diff --git a/src/include/postmaster/bgworker_internals.h b/src/include/postmaster/bgworker_internals.h index f7e24664d5..366bbcf1e3 100644 --- a/src/include/postmaster/bgworker_internals.h +++ b/src/include/postmaster/bgworker_internals.h @@ -43,6 +43,7 @@ typedef struct RegisteredBgWorker } RegisteredBgWorker; extern slist_head BackgroundWorkerList; +extern RegisteredBgWorker *CurrentBgWorker; extern Size BackgroundWorkerShmemSize(void); extern void BackgroundWorkerShmemInit(void); @@ -54,7 +55,11 @@ extern void BackgroundWorkerStopNotifications(pid_t pid); extern void ResetBackgroundWorkerCrashTimes(void); /* Function to start a background worker, called from postmaster.c */ -extern void StartBackgroundWorker(void) pg_attribute_noreturn(); +extern int BgWorkerPrep(int argc, char *argv[]); +extern void BackgroundWorkerMain(int argc, char *argv[]); +extern void BackgroundWorkerParentMain(int argc, char *argv[]); +extern bool BackgroundWorkerCleanup(int argc, char *argv[]); + #ifdef EXEC_BACKEND extern BackgroundWorker *BackgroundWorkerEntry(int slotno); diff --git a/src/include/postmaster/bgwriter.h b/src/include/postmaster/bgwriter.h index 0a5708b32e..6fb1ee1d6d 100644 --- a/src/include/postmaster/bgwriter.h +++ b/src/include/postmaster/bgwriter.h @@ -27,8 +27,8 @@ extern int CheckPointTimeout; extern int CheckPointWarning; extern double CheckPointCompletionTarget; -extern void BackgroundWriterMain(void) pg_attribute_noreturn(); -extern void CheckpointerMain(void) pg_attribute_noreturn(); +extern void BackgroundWriterMain(int argc, char *argv[]) pg_attribute_noreturn(); +extern void CheckpointerMain(int argc, char *argv[]) pg_attribute_noreturn(); extern void RequestCheckpoint(int flags); extern void CheckpointWriteDelay(int flags, double progress); diff --git a/src/include/postmaster/fork_process.h b/src/include/postmaster/fork_process.h index 8cb568fc63..a42f859a08 100644 --- a/src/include/postmaster/fork_process.h +++ b/src/include/postmaster/fork_process.h @@ -12,6 +12,12 @@ #ifndef FORK_PROCESS_H #define FORK_PROCESS_H +#include "postmaster.h" + extern pid_t fork_process(void); +#ifdef EXEC_BACKEND +extern pid_t postmaster_forkexec(int argc, char *argvp[]); +#endif + #endif /* FORK_PROCESS_H */ diff --git a/src/include/postmaster/pgarch.h b/src/include/postmaster/pgarch.h index b3200874ca..ba5b012839 100644 --- a/src/include/postmaster/pgarch.h +++ b/src/include/postmaster/pgarch.h @@ -26,14 +26,9 @@ #define MAX_XFN_CHARS 40 #define VALID_XFN_CHARS "0123456789ABCDEF.history.backup.partial" -/* ---------- - * Functions called from postmaster - * ---------- - */ -extern int pgarch_start(void); -#ifdef EXEC_BACKEND +/* Archiver subprocess startup functions */ +extern int PgArchiverPrep(int argc, char *argv[]); extern void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn(); -#endif #endif /* _PGARCH_H */ diff --git a/src/include/postmaster/postmaster.h b/src/include/postmaster/postmaster.h index babc87dfc9..d4c0f46220 100644 --- a/src/include/postmaster/postmaster.h +++ b/src/include/postmaster/postmaster.h @@ -13,6 +13,41 @@ #ifndef _POSTMASTER_H #define _POSTMASTER_H +#include "lib/ilist.h" + +/* + * List of active backends (or child processes anyway; we don't actually + * know whether a given child has become a backend or is still in the + * authorization phase). This is used mainly to keep track of how many + * children we have and send them appropriate signals when necessary. + * + * "Special" children such as the startup, bgwriter and autovacuum launcher + * tasks are not in this list. Autovacuum worker and walsender are in it. + * Also, "dead_end" children are in it: these are children launched just for + * the purpose of sending a friendly rejection message to a would-be client. + * We must track them because they are attached to shared memory, but we know + * they will never become live backends. dead_end children are not assigned a + * PMChildSlot. + * + * Background workers are in this list, too. + */ +typedef struct bkend +{ + pid_t pid; /* process id of backend */ + int32 cancel_key; /* cancel key for cancels for this backend */ + int child_slot; /* PMChildSlot for this backend, if any */ + + /* + * Flavor of backend or auxiliary process. Note that BACKEND_TYPE_WALSND + * backends initially announce themselves as BACKEND_TYPE_NORMAL, so if + * bkend_type is normal, you should check for a recent transition. + */ + int bkend_type; + bool dead_end; /* is it going to send an error and quit? */ + bool bgworker_notify; /* gets bgworker start/stop notifications */ + dlist_node elem; /* list link in BackendList */ +} Backend; + /* GUC options */ extern bool EnableSSL; extern int ReservedBackends; @@ -47,21 +82,31 @@ extern int postmaster_alive_fds[2]; extern PGDLLIMPORT const char *progname; extern void PostmasterMain(int argc, char *argv[]) pg_attribute_noreturn(); -extern void ClosePostmasterPorts(bool am_syslogger); +extern void ClosePostmasterPorts(void); extern void InitProcessGlobals(void); extern int MaxLivePostmasterChildren(void); extern bool PostmasterMarkPIDForWorkerNotify(int); +extern bool RandomCancelKey(int32 *cancel_key); + +extern int BackendPrep(int argc, char *argv[]); +extern void BackendMain(int argc, char *argv[]); +extern void BackendParentMain(int argc, char *argv[]); +extern bool BackendCleanup(int argc, char *argv[]); +extern void BackgroundWorkerMain(int argc, char *argv[]); + #ifdef EXEC_BACKEND -extern pid_t postmaster_forkexec(int argc, char *argv[]); extern void SubPostmasterMain(int argc, char *argv[]) pg_attribute_noreturn(); extern Size ShmemBackendArraySize(void); extern void ShmemBackendArrayAllocation(void); +extern void ShmemBackendArrayAdd(Backend *bn); #endif +extern PGDLLIMPORT dlist_head BackendList; + /* * Note: MAX_BACKENDS is limited to 2^18-1 because that's the width reserved * for buffer references in buf_internals.h. This limitation could be lifted @@ -74,4 +119,17 @@ extern void ShmemBackendArrayAllocation(void); */ #define MAX_BACKENDS 0x3FFFF +/* + * Possible types of a backend. Beyond being the possible bkend_type values in + * struct bkend, these are OR-able request flag bits for SignalSomeChildren() + * and CountChildren(). + */ +#define BACKEND_TYPE_NORMAL 0x0001 /* normal backend */ +#define BACKEND_TYPE_AUTOVAC 0x0002 /* autovacuum worker process */ +#define BACKEND_TYPE_WALSND 0x0004 /* walsender process */ +#define BACKEND_TYPE_BGWORKER 0x0008 /* bgworker process */ +#define BACKEND_TYPE_ALL 0x000F /* OR of all the above */ + +#define BACKEND_TYPE_WORKER (BACKEND_TYPE_AUTOVAC | BACKEND_TYPE_BGWORKER) + #endif /* _POSTMASTER_H */ diff --git a/src/include/postmaster/startup.h b/src/include/postmaster/startup.h index 9f59c1ffa3..14a646136f 100644 --- a/src/include/postmaster/startup.h +++ b/src/include/postmaster/startup.h @@ -13,10 +13,13 @@ #define _STARTUP_H extern void HandleStartupProcInterrupts(void); -extern void StartupProcessMain(void) pg_attribute_noreturn(); extern void PreRestoreCommand(void); extern void PostRestoreCommand(void); extern bool IsPromoteTriggered(void); extern void ResetPromoteTriggered(void); +/* Startup subprocess functions */ +extern void StartupProcessMain(int argc, char *argv[]) pg_attribute_noreturn(); +extern bool StartupCleanup(int argc, char *argv[]); + #endif /* _STARTUP_H */ diff --git a/src/include/postmaster/subprocess.h b/src/include/postmaster/subprocess.h new file mode 100644 index 0000000000..001f8455a9 --- /dev/null +++ b/src/include/postmaster/subprocess.h @@ -0,0 +1,62 @@ +/*------------------------------------------------------------------------- + * + * subprocess.h + * + * Copyright (c) 1996-2020, PostgreSQL Global Development Group + * + * src/include/postmaster/subprocess.h + * + *------------------------------------------------------------------------- + */ +#ifndef SUBPROCESS_H +#define SUBPROCESS_H + +#include "postmaster.h" + +typedef enum +{ + NoProcessType = -1, + CheckerType = 0, + BootstrapType, + StartupType, + BgWriterType, + CheckpointerType, + WalWriterType, + WalReceiverType, /* end of Auxiliary Process Forks */ + AutoVacLauncherType, + AutoVacWorkerType, + PgstatCollectorType, + PgArchiverType, + SysLoggerType, + BgWorkerType, + ClientBackendType, + + NUMSUBPROCESSTYPES /* Must be last! */ +} SubprocessType; + +typedef int (*SubprocessInit) (int argc, char *argv[]); +typedef void (*SubprocessEntryPoint) (int argc, char *argv[]); +typedef bool (*SubprocessCleanup) (int argc, char *argv[]); +typedef void (*SubprocessParent) (int argc, char *argv[]); + +/* Current subprocess initializer */ +extern void InitializeMySubprocess(SubprocessType type); + +typedef struct PgSubprocess +{ + const char *progname; + const char *name; + const char *desc; + bool needs_shmem; + bool needs_aux_proc; + bool needs_full_proc; + bool keep_postmaster_memcontext; + SubprocessInit init; + SubprocessEntryPoint entrypoint; + SubprocessCleanup cleanup; + SubprocessParent parent_main; +} PgSubprocess; + +extern PGDLLIMPORT PgSubprocess *MySubprocess; + +#endif /* SUBPROCESS_H */ diff --git a/src/include/postmaster/syslogger.h b/src/include/postmaster/syslogger.h index 9b7a386dbd..4cbed2ec40 100644 --- a/src/include/postmaster/syslogger.h +++ b/src/include/postmaster/syslogger.h @@ -14,7 +14,6 @@ #include <limits.h> /* for PIPE_BUF */ - /* * Primitive protocol structure for writing to syslogger pipe(s). The idea * here is to divide long messages into chunks that are not more than @@ -83,9 +82,10 @@ extern int SysLogger_Start(void); extern void write_syslogger_file(const char *buffer, int count, int dest); -#ifdef EXEC_BACKEND +/* Subprocess startup functions */ +extern int SysLoggerPrep(int argc, char *argv[]); +extern void SysLoggerParentMain(int argc, char *argv[]); extern void SysLoggerMain(int argc, char *argv[]) pg_attribute_noreturn(); -#endif extern bool CheckLogrotateSignal(void); extern void RemoveLogrotateSignalFiles(void); diff --git a/src/include/postmaster/walwriter.h b/src/include/postmaster/walwriter.h index 011dc26a6e..596c437c54 100644 --- a/src/include/postmaster/walwriter.h +++ b/src/include/postmaster/walwriter.h @@ -16,6 +16,6 @@ extern int WalWriterDelay; extern int WalWriterFlushAfter; -extern void WalWriterMain(void) pg_attribute_noreturn(); +extern void WalWriterMain(int argc, char *argv[]) pg_attribute_noreturn(); #endif /* _WALWRITER_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index e08afc6548..2025f19ab3 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -311,7 +311,7 @@ walrcv_clear_result(WalRcvExecResult *walres) } /* prototypes for functions in walreceiver.c */ -extern void WalReceiverMain(void) pg_attribute_noreturn(); +extern void WalReceiverMain(int argc, char *argv[]); extern void ProcessWalRcvInterrupts(void); /* prototypes for functions in walreceiverfuncs.c */ -- 2.21.0
