From cb67895a2227a19774a9093e76c16957bf8b746b Mon Sep 17 00:00:00 2001 From: Ashutosh Sharma Date: Wed, 19 Aug 2026 11:41:37 +0000 Subject: [PATCH] Remove files left by crash-aborted relation creation PostgreSQL tracks files created by a transaction in backend-local pending-delete state. An immediate shutdown loses that state. Files created by transactions that recovery treats as aborted can remain on disk without a visible catalog entry. Introduce durable creation markers in pg_relcreate for transactional permanent relation storage. Store the full RelFileLocator and creator XID, format version, and CRC in each marker. Write and fsync each marker and its directory before creating the physical relation file. Mark SMGR_CREATE WAL records that require a marker, allowing redo to recreate the marker idempotently before recreating relation storage. Force a local synchronous commit for transactions that create marked storage, then remove their markers only after commit WAL is durable. For aborts, retain the marker until the checkpointer removes the SMGR unlink tombstone. This prevents stale cleanup state from deleting a new relation that reuses the same relfilenumber. At the end of recovery, reconcile markers after prepared transactions have been restored. Keep files belonging to committed or prepared transactions, and remove storage belonging to crash-aborted ones. Handle COMMIT PREPARED by removing its delete-on-abort markers. Teach initdb about pg_relcreate, document the directory, and expose marker creation in WAL descriptions. Add recovery tests covering normal commit, crash abort across a checkpoint, and prepared commit. --- doc/src/sgml/storage.sgml | 6 + src/backend/access/rmgrdesc/smgrdesc.c | 2 + src/backend/access/transam/twophase.c | 32 +++ src/backend/access/transam/xlog.c | 4 + src/backend/catalog/storage.c | 225 +++++++++++++++++- src/backend/storage/smgr/md.c | 12 +- src/bin/initdb/initdb.c | 1 + src/include/access/twophase.h | 1 + src/include/catalog/storage.h | 2 + src/include/catalog/storage_xlog.h | 3 + src/test/recovery/meson.build | 1 + .../recovery/t/056_relation_create_markers.pl | 76 ++++++ 12 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 src/test/recovery/t/056_relation_create_markers.pl diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 19924b98d71..b0a10b14238 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -106,6 +106,12 @@ Item Subdirectory containing replication slot data + + pg_relcreate + Subdirectory containing durable markers for transactional relation + creation + + pg_serial Subdirectory containing information about committed serializable transactions diff --git a/src/backend/access/rmgrdesc/smgrdesc.c b/src/backend/access/rmgrdesc/smgrdesc.c index aaf1b07999d..f20c88099d6 100644 --- a/src/backend/access/rmgrdesc/smgrdesc.c +++ b/src/backend/access/rmgrdesc/smgrdesc.c @@ -29,6 +29,8 @@ smgr_desc(StringInfo buf, XLogReaderState *record) appendStringInfoString(buf, relpathperm(xlrec->rlocator, xlrec->forkNum).str); + if (xlrec->createMarker) + appendStringInfoString(buf, " with creation marker"); } else if (info == XLOG_SMGR_TRUNCATE) { diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 48e478a4ecb..4ea721e7439 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1498,6 +1498,32 @@ StandbyTransactionIdIsPrepared(TransactionId xid) return result; } +bool +TwoPhaseTransactionIdIsPrepared(TransactionId xid) +{ + TransactionId topxid; + bool result = false; + + Assert(TransactionIdIsValid(xid)); + topxid = SubTransGetTopmostTransaction(xid); + + LWLockAcquire(TwoPhaseStateLock, LW_SHARED); + for (int i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + GlobalTransaction gxact = TwoPhaseState->prepXacts[i]; + + if (gxact->valid && + TransactionIdEquals(XidFromFullTransactionId(gxact->fxid), topxid)) + { + result = true; + break; + } + } + LWLockRelease(TwoPhaseStateLock); + + return result; +} + /* * FinishPreparedTransaction: execute COMMIT PREPARED or ROLLBACK PREPARED */ @@ -1625,6 +1651,12 @@ FinishPreparedTransaction(const char *gid, bool isCommit) /* Make sure files supposed to be dropped are dropped */ DropRelationFiles(delrels, ndelrels, false); + if (isCommit) + { + for (int i = 0; i < hdr->nabortrels; i++) + RelationCreateMarkerCleanup(&abortrels[i]); + } + if (isCommit) pgstat_execute_transactional_drops(hdr->ncommitstats, commitstats, false); else diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index de4c96e135f..f0cb0ff5200 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -67,6 +67,7 @@ #include "catalog/catversion.h" #include "catalog/pg_control.h" #include "catalog/pg_database.h" +#include "catalog/storage.h" #include "common/controldata_utils.h" #include "common/file_utils.h" #include "executor/instrument.h" @@ -6715,6 +6716,9 @@ StartupXLOG(void) if (standbyState != STANDBY_DISABLED) ShutdownRecoveryTransactionEnvironment(); + if (performedWalRecovery) + RelationCreateMarkerCleanupAtEndOfRecovery(); + /* * If there were cascading standby servers connected to us, nudge any wal * sender processes to notice that we've been promoted. diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index e443a4993c5..a9c40cf2092 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -20,6 +20,8 @@ #include "postgres.h" #include "access/visibilitymap.h" +#include "access/transam.h" +#include "access/twophase.h" #include "access/xact.h" #include "access/xlog.h" #include "access/xloginsert.h" @@ -28,7 +30,9 @@ #include "catalog/storage_xlog.h" #include "miscadmin.h" #include "pgstat.h" +#include "port/pg_crc32c.h" #include "storage/bulk_write.h" +#include "storage/fd.h" #include "storage/freespace.h" #include "storage/proc.h" #include "storage/smgr.h" @@ -39,6 +43,179 @@ /* GUC variables */ int wal_skip_threshold = 2048; /* in kilobytes */ +#define RELATION_CREATE_MARKER_DIR "pg_relcreate" +#define RELATION_CREATE_MARKER_MAGIC 0x52434D4B +#define RELATION_CREATE_MARKER_VERSION 1 + +typedef struct RelationCreateMarker +{ + uint32 magic; + uint32 version; + RelFileLocator rlocator; + TransactionId xid; + pg_crc32c crc; +} RelationCreateMarker; + +static void +relation_create_marker_path(char *path, Size size, + const RelFileLocator *rlocator) +{ + snprintf(path, size, RELATION_CREATE_MARKER_DIR "/%u_%u_%u", + rlocator->spcOid, rlocator->dbOid, rlocator->relNumber); +} + +static void +create_relation_create_marker(const RelFileLocator *rlocator, + TransactionId xid) +{ + RelationCreateMarker marker; + RelationCreateMarker existing; + char path[MAXPGPATH]; + int fd; + int save_errno; + + relation_create_marker_path(path, sizeof(path), rlocator); + marker.magic = RELATION_CREATE_MARKER_MAGIC; + marker.version = RELATION_CREATE_MARKER_VERSION; + marker.rlocator = *rlocator; + marker.xid = xid; + INIT_CRC32C(marker.crc); + COMP_CRC32C(marker.crc, &marker, offsetof(RelationCreateMarker, crc)); + FIN_CRC32C(marker.crc); + + fd = OpenTransientFile(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY); + if (fd < 0 && errno == EEXIST) + { + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd >= 0 && + read(fd, &existing, sizeof(existing)) == sizeof(existing) && + existing.magic == marker.magic && + existing.version == marker.version && + RelFileLocatorEquals(existing.rlocator, marker.rlocator) && + TransactionIdEquals(existing.xid, marker.xid) && + EQ_CRC32C(existing.crc, marker.crc)) + { + CloseTransientFile(fd); + return; + } + if (fd >= 0) + CloseTransientFile(fd); + errno = EEXIST; + } + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create relation creation marker \"%s\": %m", + path))); + + if (write(fd, &marker, sizeof(marker)) != sizeof(marker) || + pg_fsync(fd) != 0) + { + save_errno = errno; + CloseTransientFile(fd); + errno = save_errno; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write relation creation marker \"%s\": %m", + path))); + } + + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close relation creation marker \"%s\": %m", + path))); + + fsync_fname(RELATION_CREATE_MARKER_DIR, true); +} + +void +RelationCreateMarkerCleanup(const RelFileLocator *rlocator) +{ + char path[MAXPGPATH]; + + relation_create_marker_path(path, sizeof(path), rlocator); + (void) durable_unlink(path, WARNING); +} + +void +RelationCreateMarkerCleanupAtEndOfRecovery(void) +{ + DIR *dir; + struct dirent *de; + + dir = AllocateDir(RELATION_CREATE_MARKER_DIR); + if (dir == NULL) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not open relation creation marker directory \"%s\": %m", + RELATION_CREATE_MARKER_DIR))); + + while ((de = ReadDir(dir, RELATION_CREATE_MARKER_DIR)) != NULL) + { + RelationCreateMarker marker; + char path[MAXPGPATH]; + char expected_path[MAXPGPATH]; + int fd; + pg_crc32c crc; + + if (de->d_name[0] == '.') + continue; + + snprintf(path, sizeof(path), RELATION_CREATE_MARKER_DIR "/%s", + de->d_name); + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0 || read(fd, &marker, sizeof(marker)) != sizeof(marker)) + { + if (fd >= 0) + CloseTransientFile(fd); + ereport(FATAL, + (errcode_for_file_access(), + errmsg("invalid relation creation marker \"%s\"", path))); + } + + INIT_CRC32C(crc); + COMP_CRC32C(crc, &marker, offsetof(RelationCreateMarker, crc)); + FIN_CRC32C(crc); + relation_create_marker_path(expected_path, sizeof(expected_path), + &marker.rlocator); + if (marker.magic != RELATION_CREATE_MARKER_MAGIC || + marker.version != RELATION_CREATE_MARKER_VERSION || + !TransactionIdIsValid(marker.xid) || + !EQ_CRC32C(crc, marker.crc) || strcmp(path, expected_path) != 0) + { + CloseTransientFile(fd); + ereport(FATAL, + (errcode_for_file_access(), + errmsg("invalid relation creation marker \"%s\"", path))); + } + if (CloseTransientFile(fd) != 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not close relation creation marker \"%s\": %m", + path))); + + if (TransactionIdDidCommit(marker.xid)) + { + RelationCreateMarkerCleanup(&marker.rlocator); + continue; + } + if (TwoPhaseTransactionIdIsPrepared(marker.xid)) + continue; + + { + SMgrRelation srel = smgropen(marker.rlocator, + INVALID_PROC_NUMBER); + + smgrdounlinkall(&srel, 1, true); + smgrclose(srel); + RelationCreateMarkerCleanup(&marker.rlocator); + } + } + + FreeDir(dir); +} + /* * We keep a list of all relations (represented as RelFileLocator values) * that have been created or deleted in the current transaction. When @@ -63,6 +240,7 @@ typedef struct PendingRelDelete { RelFileLocator rlocator; /* relation that may need to be deleted */ ProcNumber procNumber; /* INVALID_PROC_NUMBER if not a temp rel */ + TransactionId createXid; /* XID stored in the creation marker */ bool atCommit; /* T=delete at commit; F=delete at abort */ int nestLevel; /* xact nesting level of request */ struct PendingRelDelete *next; /* linked-list link */ @@ -124,6 +302,7 @@ RelationCreateStorage(RelFileLocator rlocator, char relpersistence, { SMgrRelation srel; ProcNumber procNumber; + TransactionId createXid = InvalidTransactionId; bool needs_wal; Assert(!IsInParallelMode()); /* couldn't update pendingSyncHash */ @@ -148,9 +327,18 @@ RelationCreateStorage(RelFileLocator rlocator, char relpersistence, } srel = smgropen(rlocator, procNumber); + + if (needs_wal && register_delete) + { + createXid = log_smgrcreate_marker(&srel->smgr_rlocator.locator, + MAIN_FORKNUM); + create_relation_create_marker(&rlocator, createXid); + ForceSyncCommit(); + } + smgrcreate(srel, MAIN_FORKNUM, false); - if (needs_wal) + if (needs_wal && !register_delete) log_smgrcreate(&srel->smgr_rlocator.locator, MAIN_FORKNUM); /* @@ -165,6 +353,7 @@ RelationCreateStorage(RelFileLocator rlocator, char relpersistence, MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete)); pending->rlocator = rlocator; pending->procNumber = procNumber; + pending->createXid = createXid; pending->atCommit = false; /* delete if abort */ pending->nestLevel = GetCurrentTransactionNestLevel(); pending->next = pendingDeletes; @@ -186,17 +375,35 @@ RelationCreateStorage(RelFileLocator rlocator, char relpersistence, void log_smgrcreate(const RelFileLocator *rlocator, ForkNumber forkNum) { - xl_smgr_create xlrec; + xl_smgr_create xlrec = {0}; /* * Make an XLOG entry reporting the file creation. */ xlrec.rlocator = *rlocator; xlrec.forkNum = forkNum; + xlrec.createMarker = false; + + XLogBeginInsert(); + XLogRegisterData(&xlrec, sizeof(xlrec)); + XLogInsert(RM_SMGR_ID, XLOG_SMGR_CREATE | XLR_SPECIAL_REL_UPDATE); +} + +TransactionId +log_smgrcreate_marker(const RelFileLocator *rlocator, ForkNumber forkNum) +{ + xl_smgr_create xlrec = {0}; + TransactionId xid = GetCurrentTransactionId(); + + xlrec.rlocator = *rlocator; + xlrec.forkNum = forkNum; + xlrec.createMarker = true; XLogBeginInsert(); XLogRegisterData(&xlrec, sizeof(xlrec)); XLogInsert(RM_SMGR_ID, XLOG_SMGR_CREATE | XLR_SPECIAL_REL_UPDATE); + + return xid; } /* @@ -213,6 +420,7 @@ RelationDropStorage(Relation rel) MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete)); pending->rlocator = rel->rd_locator; pending->procNumber = rel->rd_backend; + pending->createXid = InvalidTransactionId; pending->atCommit = true; /* delete if commit */ pending->nestLevel = GetCurrentTransactionNestLevel(); pending->next = pendingDeletes; @@ -262,6 +470,9 @@ RelationPreserveStorage(RelFileLocator rlocator, bool atCommit) if (RelFileLocatorEquals(rlocator, pending->rlocator) && pending->atCommit == atCommit) { + if (!atCommit && TransactionIdIsValid(pending->createXid)) + RelationCreateMarkerCleanup(&pending->rlocator); + /* unlink and delete list entry */ if (prev) prev->next = next; @@ -717,6 +928,8 @@ smgrDoPendingDeletes(bool isCommit) srels[nrels++] = srel; } + else if (isCommit && TransactionIdIsValid(pending->createXid)) + RelationCreateMarkerCleanup(&pending->rlocator); /* must explicitly free the list entry */ pfree(pending); /* prev does not change */ @@ -990,6 +1203,14 @@ smgr_redo(XLogReaderState *record) { xl_smgr_create *xlrec = (xl_smgr_create *) XLogRecGetData(record); SMgrRelation reln; + TransactionId xid = XLogRecGetXid(record); + + if (xlrec->createMarker) + { + if (!TransactionIdIsValid(xid)) + elog(PANIC, "relation creation marker WAL record has no transaction ID"); + create_relation_create_marker(&xlrec->rlocator, xid); + } reln = smgropen(xlrec->rlocator, INVALID_PROC_NUMBER); smgrcreate(reln, xlrec->forkNum, true); diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 780c88c0630..b0ac91154a6 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -27,6 +27,7 @@ #include #include "access/xlogutils.h" +#include "catalog/storage.h" #include "commands/tablespace.h" #include "common/file_utils.h" #include "miscadmin.h" @@ -1957,6 +1958,8 @@ int mdunlinkfiletag(const FileTag *ftag, char *path) { RelPathStr p; + int result; + int save_errno; /* We only unlink tombstone files through this mechanism */ Assert(ftag->forknum == MAIN_FORKNUM && ftag->segno == 0); @@ -1966,7 +1969,14 @@ mdunlinkfiletag(const FileTag *ftag, char *path) strlcpy(path, p.str, MAXPGPATH); /* Try to unlink the file. */ - return unlink(path); + result = unlink(path); + save_errno = errno; + + if (result == 0 || save_errno == ENOENT) + RelationCreateMarkerCleanup(&ftag->rlocator); + + errno = save_errno; + return result; } /* diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index b3d496372ad..f2ff108749c 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -236,6 +236,7 @@ static const char *const subdirs[] = { "pg_commit_ts", "pg_dynshmem", "pg_notify", + "pg_relcreate", "pg_serial", "pg_snapshots", "pg_subtrans", diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h index 1d2ff42c9b7..152b1e0d2f6 100644 --- a/src/include/access/twophase.h +++ b/src/include/access/twophase.h @@ -48,6 +48,7 @@ extern GlobalTransaction MarkAsPreparing(FullTransactionId fxid, const char *gid extern void StartPrepare(GlobalTransaction gxact); extern void EndPrepare(GlobalTransaction gxact); extern bool StandbyTransactionIdIsPrepared(TransactionId xid); +extern bool TwoPhaseTransactionIdIsPrepared(TransactionId xid); extern TransactionId PrescanPreparedTransactions(TransactionId **xids_p, int *nxids_p); diff --git a/src/include/catalog/storage.h b/src/include/catalog/storage.h index 70f619a6d6f..6cf7ca5762a 100644 --- a/src/include/catalog/storage.h +++ b/src/include/catalog/storage.h @@ -27,6 +27,8 @@ extern SMgrRelation RelationCreateStorage(RelFileLocator rlocator, bool register_delete); extern void RelationDropStorage(Relation rel); extern void RelationPreserveStorage(RelFileLocator rlocator, bool atCommit); +extern void RelationCreateMarkerCleanup(const RelFileLocator *rlocator); +extern void RelationCreateMarkerCleanupAtEndOfRecovery(void); extern void RelationPreTruncate(Relation rel); extern void RelationTruncate(Relation rel, BlockNumber nblocks); extern void RelationCopyStorage(SMgrRelation src, SMgrRelation dst, diff --git a/src/include/catalog/storage_xlog.h b/src/include/catalog/storage_xlog.h index c1b2f736669..4924f050313 100644 --- a/src/include/catalog/storage_xlog.h +++ b/src/include/catalog/storage_xlog.h @@ -34,6 +34,7 @@ typedef struct xl_smgr_create { RelFileLocator rlocator; ForkNumber forkNum; + bool createMarker; } xl_smgr_create; /* flags for xl_smgr_truncate */ @@ -51,6 +52,8 @@ typedef struct xl_smgr_truncate } xl_smgr_truncate; extern void log_smgrcreate(const RelFileLocator *rlocator, ForkNumber forkNum); +extern TransactionId log_smgrcreate_marker(const RelFileLocator *rlocator, + ForkNumber forkNum); extern void smgr_redo(XLogReaderState *record); extern void smgr_desc(StringInfo buf, XLogReaderState *record); diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 39ec8c4946d..b8e43847c2e 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -64,6 +64,7 @@ tests += { 't/053_standby_login_event_trigger.pl', 't/054_unlogged_sequence_promotion.pl', 't/055_cascade_reconnect.pl', + 't/056_relation_create_markers.pl', ], }, } diff --git a/src/test/recovery/t/056_relation_create_markers.pl b/src/test/recovery/t/056_relation_create_markers.pl new file mode 100644 index 00000000000..a1b1ef74c8c --- /dev/null +++ b/src/test/recovery/t/056_relation_create_markers.pl @@ -0,0 +1,76 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test cleanup of permanent relation files created by transactions that are +# still in progress when the server crashes. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('relation_create_markers'); +$node->init(); +$node->append_conf('postgresql.conf', 'max_prepared_transactions = 10'); +$node->start(); + +my $marker_dir = $node->data_dir . '/pg_relcreate'; + +$node->safe_psql('postgres', 'CREATE TABLE committed_relation (a int)'); +is(scalar(grep { $_ ne '.' && $_ ne '..' } slurp_dir($marker_dir)), + 0, 'committed relation leaves no creation marker'); + +my $session = $node->background_psql('postgres'); +$session->query_safe('BEGIN'); +my $relation_path = $session->query_safe( + 'CREATE TABLE crash_aborted_relation (a int); ' + . q{SELECT pg_relation_filepath('crash_aborted_relation')}); + +ok(-f $node->data_dir . '/' . $relation_path, + 'uncommitted relation file exists before crash'); +is(scalar(grep { $_ ne '.' && $_ ne '..' } slurp_dir($marker_dir)), + 1, 'uncommitted relation has a creation marker'); + +# Move the redo pointer past the creation record. Recovery therefore needs +# the persistent marker; replay-local tracking of the create record is not +# sufficient. +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->stop('immediate'); +$node->start(); + +is($node->safe_psql('postgres', + q{SELECT to_regclass('crash_aborted_relation') IS NULL}), + 't', 'crash-aborted relation is absent from the catalog'); +ok(!-e $node->data_dir . '/' . $relation_path, + 'crash-aborted relation file is removed during recovery'); +is(scalar(grep { $_ ne '.' && $_ ne '..' } slurp_dir($marker_dir)), + 0, 'processed creation marker is removed'); + +my $prepared_path = $node->safe_psql( + 'postgres', + q{BEGIN; +CREATE TABLE prepared_relation (a int); +SELECT pg_relation_filepath('prepared_relation'); +PREPARE TRANSACTION 'relation_create_marker';}); +ok(-f $node->data_dir . '/' . $prepared_path, + 'prepared relation file exists'); +is(scalar(grep { $_ ne '.' && $_ ne '..' } slurp_dir($marker_dir)), + 1, 'prepared relation retains its creation marker'); + +$node->stop('immediate'); +$node->start(); + +ok(-f $node->data_dir . '/' . $prepared_path, + 'prepared relation file survives recovery'); +is(scalar(grep { $_ ne '.' && $_ ne '..' } slurp_dir($marker_dir)), + 1, 'recovery retains prepared relation marker'); +$node->safe_psql('postgres', + q{COMMIT PREPARED 'relation_create_marker'}); +is($node->safe_psql('postgres', + q{SELECT to_regclass('prepared_relation') IS NOT NULL}), + 't', 'committed prepared relation is visible'); +is(scalar(grep { $_ ne '.' && $_ ne '..' } slurp_dir($marker_dir)), + 0, 'commit prepared removes relation marker'); + +$node->stop(); +done_testing(); -- 2.43.0