From 27f371644ff0d6a02cf981f0045c17d4a21c94ae Mon Sep 17 00:00:00 2001 From: Ayush Tiwari Date: Fri, 25 Sep 2026 01:38:41 +0530 Subject: [PATCH v1] Clear FatalError when relaunching the startup process After a crash, the postmaster reinitializes shared memory and launches a new startup process, but FatalError stays set until WAL redo starts. A smart or fast shutdown in that window hangs: the postmaster sends SIGTERM to the checkpointer and I/O workers, which ignore it, and waits for them in PM_WAIT_BACKENDS indefinitely. Clear FatalError when the new startup process is launched. The old children are gone by then, so shutdown requests and new child failures can be handled as during an initial startup. This makes the special startup-failure handling added by ead8f696b7c unnecessary, so remove it and restore the HandleFatalError() assertion that it relaxed. Suggested-by: Noah Misch --- src/backend/postmaster/postmaster.c | 44 +++++------------ src/test/recovery/meson.build | 1 + .../recovery/t/058_shutdown_crash_restart.pl | 48 +++++++++++++++++++ src/test/recovery/t/wait_for_shutdown | 19 ++++++++ 4 files changed, 80 insertions(+), 32 deletions(-) create mode 100644 src/test/recovery/t/058_shutdown_crash_restart.pl create mode 100644 src/test/recovery/t/wait_for_shutdown diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index ef300a6c45a..f5ef5c5314e 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -288,7 +288,7 @@ static StartupStatusEnum StartupStatus = STARTUP_NOT_RUNNING; static int Shutdown = NoShutdown; -static bool FatalError = false; /* T if recovering from backend crash */ +static bool FatalError = false; /* T while handling a fatal error */ /* * We use a simple state machine to control startup, shutdown, and @@ -329,9 +329,8 @@ static bool FatalError = false; /* T if recovering from backend crash */ * states later than PM_RUN --- Shutdown and FatalError must be consulted * to find that out. FatalError is never true in PM_RECOVERY, PM_HOT_STANDBY, * or PM_RUN states, nor in PM_WAIT_XLOG_SHUTDOWN states (because we don't - * enter those states when trying to recover from a crash). It can be true in - * PM_STARTUP state, because we don't clear it until we've successfully - * started WAL redo. + * enter those states when trying to recover from a crash). It is cleared + * when the startup process is relaunched after reinitializing shared memory. */ typedef enum { @@ -2332,25 +2331,8 @@ process_pm_child_exit(void) } else StartupStatus = STARTUP_CRASHED; - - /* - * If FatalError is already set, we are reinitializing after a - * previous crash, and HandleChildCrash() would do nothing, - * leaving the state machine stuck at PM_STARTUP. Give up, - * signal the remaining children and head for PM_NO_CHILDREN, - * where STARTUP_CRASHED makes us exit. - */ - if (StartupStatus == STARTUP_CRASHED && - FatalError && Shutdown != ImmediateShutdown) - { - LogChildExit(LOG, _("startup process"), pid, exitstatus); - ereport(LOG, - (errmsg("aborting startup due to startup process failure"))); - HandleFatalError(PMQUIT_FOR_CRASH, true); - } - else - HandleChildCrash(pid, exitstatus, - _("startup process")); + HandleChildCrash(pid, exitstatus, + _("startup process")); continue; } @@ -2740,13 +2722,15 @@ CleanupBackend(PMChild *bp, * happened. Commonly the caller will have logged the reason for entering * FatalError state. * - * This should only be called when not already in ImmediateShutdown state. + * This should only be called when not already in FatalError or + * ImmediateShutdown state. */ static void HandleFatalError(QuitSignalReason reason, bool consider_sigabrt) { int sigtosend; + Assert(!FatalError); Assert(Shutdown != ImmediateShutdown); SetQuitSignalReason(reason); @@ -3280,7 +3264,8 @@ PostmasterStateMachine(void) StartupPMChild = StartChildProcess(B_STARTUP); Assert(StartupPMChild != NULL); StartupStatus = STARTUP_RUNNING; - /* crash recovery started, reset SIGKILL flag */ + /* The old children are gone; proceed as at initial startup */ + FatalError = false; AbortStartTime = 0; /* start accepting server socket connection events again */ @@ -3756,9 +3741,7 @@ process_pm_pmsignal(void) if (CheckPostmasterSignal(PMSIGNAL_RECOVERY_STARTED) && pmState == PM_STARTUP && Shutdown == NoShutdown) { - /* WAL redo has started. We're out of reinitialization. */ - FatalError = false; - AbortStartTime = 0; + /* WAL redo has started. */ reachedConsistency = false; /* @@ -4297,10 +4280,7 @@ maybe_start_bgworkers(void) TimestampTz now = 0; dlist_mutable_iter iter; - /* - * During crash recovery, we have no need to be called until the state - * transition out of recovery. - */ + /* Don't start workers until the old children have exited */ if (FatalError) { StartWorkerNeeded = false; diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index ebb12dd8766..aee97e4da2a 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -66,6 +66,7 @@ tests += { 't/055_cascade_reconnect.pl', 't/056_standby_snapshot_export.pl', 't/057_snapshot_commit_race.pl', + 't/058_shutdown_crash_restart.pl', ], }, } diff --git a/src/test/recovery/t/058_shutdown_crash_restart.pl b/src/test/recovery/t/058_shutdown_crash_restart.pl new file mode 100644 index 00000000000..c7d7fe39d1e --- /dev/null +++ b/src/test/recovery/t/058_shutdown_crash_restart.pl @@ -0,0 +1,48 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test fast shutdown during crash restart, before WAL redo has started. + +use strict; +use warnings FATAL => 'all'; +use FindBin; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('node'); +$node->init(allows_streaming => 1); + +# Make the restarted startup process wait in restore_command until shutdown. +my $perlbin = $^X; +$perlbin =~ s!\\!/!g if $windows_os; +my $logfile = $node->logfile; +$logfile =~ s!\\!/!g if $windows_os; +my $timeout = $PostgreSQL::Test::Utils::timeout_default; +$node->append_conf( + 'postgresql.conf', qq{ +restart_after_crash = on +restore_command = '"$perlbin" "$FindBin::RealBin/wait_for_shutdown" "$logfile" $timeout' +}); +$node->start; + +$node->poll_query_until( + 'postgres', + q{SELECT count(*) = 1 FROM pg_stat_activity + WHERE backend_type = 'background writer'} +) or die 'background writer did not start'; +my $pid = $node->safe_psql('postgres', + "SELECT pid FROM pg_stat_activity WHERE backend_type = 'background writer'" +); +$node->set_standby_mode; +system_or_bail('pg_ctl', 'kill', 'QUIT', $pid); +$node->wait_for_log(qr/restore_command waiting for shutdown/); + +ok( $node->stop('fast', fail_ok => 1, timeout => $timeout), + 'fast shutdown completes during crash restart'); +# If the helper gave up, startup failed and the shutdown above proves nothing. +unlike( + slurp_file($node->logfile), + qr/timed out waiting for shutdown request/, + 'restore_command did not time out'); + +done_testing(); diff --git a/src/test/recovery/t/wait_for_shutdown b/src/test/recovery/t/wait_for_shutdown new file mode 100644 index 00000000000..8c9ceb5b70a --- /dev/null +++ b/src/test/recovery/t/wait_for_shutdown @@ -0,0 +1,19 @@ +#!/usr/bin/perl + +# restore_command helper: wait until the server log shows a shutdown request. + +use strict; +use warnings FATAL => 'all'; +use Time::HiRes qw(usleep); + +my ($logfile, $timeout) = @ARGV; + +print STDERR "restore_command waiting for shutdown\n"; +for (1 .. $timeout * 10) +{ + open my $fh, '<', $logfile or die "could not open $logfile: $!"; + exit 1 if grep { /received \w+ shutdown request/ } <$fh>; + close $fh; + usleep(100_000); +} +die "timed out waiting for shutdown request\n"; base-commit: 46024c573bcb10b323874651702581cc312f4425 -- 2.34.1