Part of the work that Thomas mentions in [1], regarding Direct I/O,
has certain requirements that pointers must be page-aligned.
I've attached a patch which implements palloc_aligned() and
MemoryContextAllocAligned() which accept an 'alignto' parameter which
must be a power-of-2 value. The memory addresses returned by these
functions will be aligned by the requested alignment.
Primarily, this work is by Andres. I took what he had and cleaned it
up, fixed a few minor bugs then implemented repalloc() and
GetMemoryChunkSpace() functionality.
The way this works is that palloc_aligned() can be called for any of
the existing MemoryContext types. What we do is perform a normal
allocation request, but we add additional bytes to the size request to
allow the proper alignment of the pointer that we return. Since we
have no control over the alignment of the return value from the
allocation requests, we must adjust the pointer returned by the
allocation function to align it to the required alignment.
When an operation such as pfree() or repalloc() is performed on a
pointer retuned by palloc_aligned(), we can't go trying to pfree the
aligned pointer as this is not the pointer that was returned by the
allocation function. To make all this work, another MemoryChunk
struct exists directly before the aligned pointer which has the
MemoryContextMethodID set to MCTX_ALIGNED_REDIRECT_ID. These
"redirection" MemoryChunks have the "block offset" set to allow the
actual MemoryChunk of the original allocation to be found. We just
subtract the number of bytes stored in the block offset, which is just
the same as how we now find the owning AllocBlock from the MemoryChunk
in aset.c. Once we do that offset calculation, we can just pfree()
the original chunk.
The 'alignto' is stored in this "redirection" MemoryChunk so that
repalloc() knows what the original alignment request was so that it
the repalloc'd chunk can be aligned by that amount too.
In the attached patch, there are not yet any users of these new 2
functions. As mentioned above, Thomas is proposing some patches to
implement Direct I/O in [1] which will use these functions. Because I
touched memory contexts last, it likely makes the most sense for me to
work on this portion of the patch.
Comments welcome.
Patch attached.
(I did rip out all the I/O specific portions from Andres' patch which
Thomas proposes as his 0002 patch. Thomas will need to rebase
(sorry)).
David
[1]
https://www.postgresql.org/message-id/ca+hukgk1x532hyqj_mzfwt0n1zt8trz980d79wbjwnt-yyl...@mail.gmail.com
diff --git a/src/backend/utils/mmgr/Makefile b/src/backend/utils/mmgr/Makefile
index 3b4cfdbd52..dae3432c98 100644
--- a/src/backend/utils/mmgr/Makefile
+++ b/src/backend/utils/mmgr/Makefile
@@ -13,6 +13,7 @@ top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
OBJS = \
+ alignedalloc.o \
aset.o \
dsa.o \
freepage.o \
diff --git a/src/backend/utils/mmgr/alignedalloc.c
b/src/backend/utils/mmgr/alignedalloc.c
new file mode 100644
index 0000000000..e581772758
--- /dev/null
+++ b/src/backend/utils/mmgr/alignedalloc.c
@@ -0,0 +1,93 @@
+/*-------------------------------------------------------------------------
+ *
+ * alignedalloc.c
+ * Allocator functions to implement palloc_aligned
+ *
+ * This is not a fully fledged MemoryContext type as there is no means to
+ * create a MemoryContext of this type. The code here only serves to allow
+ * operations such as pfree() and repalloc() to work correctly on a memory
+ * chunk that was allocated by palloc_align().
+ *
+ * Portions Copyright (c) 2017-2022, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/utils/mmgr/alignedalloc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "utils/memdebug.h"
+#include "utils/memutils_memorychunk.h"
+
+void
+AlignedAllocFree(void *pointer)
+{
+ MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
+ void *unaligned;
+
+ Assert(!MemoryChunkIsExternal(chunk));
+
+ unaligned = MemoryChunkGetBlock(chunk);
+
+ pfree(unaligned);
+}
+
+void *
+AlignedAllocRealloc(void *pointer, Size size)
+{
+ MemoryChunk *redirchunk = PointerGetMemoryChunk(pointer);
+ Size alignto = MemoryChunkGetValue(redirchunk);
+ void *unaligned = MemoryChunkGetBlock(redirchunk);
+ MemoryChunk *chunk = PointerGetMemoryChunk(unaligned);
+ Size old_size;
+ void *newptr;
+
+ /* sanity check this is a power of 2 value */
+ Assert((alignto & (alignto - 1)) == 0);
+
+ /*
+ * Determine the size of the original allocation. We can't determine
this
+ * exactly as GetMemoryChunkSpace() returns the total space used for the
+ * allocation, which for contexts like aset includes rounding up to the
+ * next power of 2. However, this value is just used to memcpy() the
old
+ * data into the new allocation, so we only need to concern ourselves
with
+ * not reading beyond the end of the original allocation's memory. The
+ * drawback here is that we may copy more bytes than we need to, which
+ * amounts only to wasted effort.
+ */
+ old_size = GetMemoryChunkSpace(unaligned) -
+ ((char *)pointer - (char *)chunk);
+
+ newptr = palloc_aligned(size, alignto, 0);
+
+ /*
+ * We may memcpy beyond the end of the orignal allocation request size,
so
+ * we must mark the entire allocation as defined.
+ */
+ VALGRIND_MAKE_MEM_DEFINED(pointer, old_size);
+ memcpy(newptr, pointer, Min(size, old_size));
+ pfree(unaligned);
+
+ return newptr;
+}
+
+MemoryContext
+AlignedAllocGetChunkContext(void *pointer)
+{
+ MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
+
+ Assert(!MemoryChunkIsExternal(chunk));
+
+ return GetMemoryChunkContext(MemoryChunkGetBlock(chunk));
+}
+
+Size
+AlignedGetChunkSpace(void *pointer)
+{
+ MemoryChunk *redirchunk = PointerGetMemoryChunk(pointer);
+ void *unaligned = MemoryChunkGetBlock(redirchunk);
+
+ return GetMemoryChunkSpace(unaligned);
+}
diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c
index f526ca82c1..cd2c43efb3 100644
--- a/src/backend/utils/mmgr/mcxt.c
+++ b/src/backend/utils/mmgr/mcxt.c
@@ -30,6 +30,7 @@
#include "utils/memdebug.h"
#include "utils/memutils.h"
#include "utils/memutils_internal.h"
+#include "utils/memutils_memorychunk.h"
static void BogusFree(void *pointer);
@@ -84,6 +85,21 @@ static const MemoryContextMethods mcxt_methods[] = {
[MCTX_SLAB_ID].check = SlabCheck,
#endif
+ /* alignedalloc.c */
+ [MCTX_ALIGNED_REDIRECT_ID].alloc = NULL, /* not required */
+ [MCTX_ALIGNED_REDIRECT_ID].free_p = AlignedAllocFree,
+ [MCTX_ALIGNED_REDIRECT_ID].realloc = AlignedAllocRealloc,
+ [MCTX_ALIGNED_REDIRECT_ID].reset = NULL, /* not required */
+ [MCTX_ALIGNED_REDIRECT_ID].delete_context = NULL, /* not required
*/
+ [MCTX_ALIGNED_REDIRECT_ID].get_chunk_context =
AlignedAllocGetChunkContext,
+ [MCTX_ALIGNED_REDIRECT_ID].get_chunk_space = AlignedGetChunkSpace,
+ [MCTX_ALIGNED_REDIRECT_ID].is_empty = NULL, /* not required */
+ [MCTX_ALIGNED_REDIRECT_ID].stats = NULL, /* not required */
+#ifdef MEMORY_CONTEXT_CHECKING
+ [MCTX_ALIGNED_REDIRECT_ID].check = NULL, /* no required */
+#endif
+
+
/*
* Unused (as yet) IDs should have dummy entries here. This allows us
to
* fail cleanly if a bogus pointer is passed to pfree or the like. It
@@ -110,11 +126,6 @@ static const MemoryContextMethods mcxt_methods[] = {
[MCTX_UNUSED4_ID].realloc = BogusRealloc,
[MCTX_UNUSED4_ID].get_chunk_context = BogusGetChunkContext,
[MCTX_UNUSED4_ID].get_chunk_space = BogusGetChunkSpace,
-
- [MCTX_UNUSED5_ID].free_p = BogusFree,
- [MCTX_UNUSED5_ID].realloc = BogusRealloc,
- [MCTX_UNUSED5_ID].get_chunk_context = BogusGetChunkContext,
- [MCTX_UNUSED5_ID].get_chunk_space = BogusGetChunkSpace,
};
/*
@@ -1298,6 +1309,92 @@ palloc_extended(Size size, int flags)
return ret;
}
+/*
+ * MemoryContextAllocAligned
+ * Allocate 'size' bytes of memory in 'context' aligned to
'alignto'
+ * bytes.
+ *
+ * 'flags' may be 0 or set the same as MemoryContextAllocExtended().
+ * 'alignto' must be a power of 2.
+ */
+void *
+MemoryContextAllocAligned(MemoryContext context,
+ Size size, Size alignto, int
flags)
+{
+ Size alloc_size;
+ void *unaligned;
+ void *aligned;
+
+ /* wouldn't make much sense to waste that much space */
+ Assert(alignto < (128 * 1024 * 1024));
+
+ /* ensure alignto is a power of 2 */
+ Assert((alignto & (alignto - 1)) == 0);
+
+ /*
+ * If the alignment requirements are less than what we already guarantee
+ * then just use the standard allocation function.
+ */
+ if (unlikely(alignto <= MAXIMUM_ALIGNOF))
+ return palloc_extended(size, flags);
+
+ /*
+ * We implement aligned pointers by simply allocating enough memory for
+ * the requested size plus the alignment and an additional MemoryChunk.
+ * This additional MemoryChunk is required for operations such as pfree
+ * when used on the pointer returned by this function. We use this
+ * "redirection" MemoryChunk in order to find the pointer to the memory
+ * that was returned by the MemoryContextAllocExtended call below. We do
+ * that by "borrowing" the block offset field and instead of using that
to
+ * find the offset into the owning block, we use it to find the original
+ * allocated address.
+ *
+ * Here we must allocate enough extra memory so that we can still align
+ * the pointer returned by MemoryContextAllocExtended and also have
enough
+ * space for the redirection MemoryChunk.
+ */
+ alloc_size = size + alignto + sizeof(MemoryChunk);
+
+ /* perform the actual allocation */
+ unaligned = MemoryContextAllocExtended(context, alloc_size, flags);
+
+ /* set the aligned pointer */
+ aligned = (void *) TYPEALIGN(alignto, (char *) unaligned +
sizeof(MemoryChunk));
+
+ /*
+ * We set the redirect MemoryChunk so that the block offset calculation
is
+ * used to point back to the 'unaligned' allocated chunk. This allows
us
+ * to use MemoryChunkGetBlock() to find the unaligned chunk when we need
+ * to perform operations such as pfree() or repalloc().
+ *
+ * We store 'alignto' in the MemoryChunk's 'value' so that we know what
+ * the alignment was set to should we ever be asked to realloc this
+ * pointer.
+ */
+ MemoryChunkSetHdrMask(PointerGetMemoryChunk(aligned), unaligned,
alignto,
+ MCTX_ALIGNED_REDIRECT_ID);
+
+ /* XXX: should we adjust valgrind state here? */
+
+ /* double check we produced a correctly aligned pointer */
+ Assert((char *) TYPEALIGN(alignto, aligned) == aligned);
+
+ return aligned;
+}
+
+/*
+ * palloc_aligned
+ * Allocate 'size' bytes returning a pointer that's aligned to the
+ * 'alignto' boundary.
+ *
+ * 'alignto' must be a power of 2.
+ */
+void *
+palloc_aligned(Size size, Size alignto, int flags)
+{
+ return MemoryContextAllocAligned(CurrentMemoryContext, size, alignto,
flags);
+}
+
/*
* pfree
* Release an allocated chunk.
@@ -1306,11 +1403,16 @@ void
pfree(void *pointer)
{
#ifdef USE_VALGRIND
+ MemoryContextMethodID method = GetMemoryChunkMethodID(pointer);
MemoryContext context = GetMemoryChunkContext(pointer);
#endif
MCXT_METHOD(pointer, free_p) (pointer);
- VALGRIND_MEMPOOL_FREE(context, pointer);
+
+#ifdef USE_VALGRIND
+ if (method != MCTX_ALIGNED_REDIRECT_ID)
+ VALGRIND_MEMPOOL_FREE(context, pointer);
+#endif
}
/*
diff --git a/src/backend/utils/mmgr/meson.build
b/src/backend/utils/mmgr/meson.build
index 641bb181ba..7cf4d6dcc8 100644
--- a/src/backend/utils/mmgr/meson.build
+++ b/src/backend/utils/mmgr/meson.build
@@ -1,4 +1,5 @@
backend_sources += files(
+ 'alignedalloc.c',
'aset.c',
'dsa.c',
'freepage.c',
diff --git a/src/include/utils/memutils_internal.h
b/src/include/utils/memutils_internal.h
index bc2cbdd506..450bcba3ed 100644
--- a/src/include/utils/memutils_internal.h
+++ b/src/include/utils/memutils_internal.h
@@ -70,6 +70,15 @@ extern void SlabStats(MemoryContext context,
extern void SlabCheck(MemoryContext context);
#endif
+/*
+ * These functions support the implementation of palloc_aligned() and are not
+ * part of a fully-fledged MemoryContext type.
+ */
+extern void AlignedAllocFree(void *pointer);
+extern void *AlignedAllocRealloc(void *pointer, Size size);
+extern MemoryContext AlignedAllocGetChunkContext(void *pointer);
+extern Size AlignedGetChunkSpace(void *pointer);
+
/*
* MemoryContextMethodID
* A unique identifier for each MemoryContext implementation which
@@ -92,8 +101,8 @@ typedef enum MemoryContextMethodID
MCTX_ASET_ID,
MCTX_GENERATION_ID,
MCTX_SLAB_ID,
- MCTX_UNUSED4_ID, /* available */
- MCTX_UNUSED5_ID /* 111 occurs in wipe_mem'd
memory */
+ MCTX_ALIGNED_REDIRECT_ID,
+ MCTX_UNUSED4_ID /* 111 occurs in wipe_mem'd
memory */
} MemoryContextMethodID;
/*
diff --git a/src/include/utils/memutils_memorychunk.h
b/src/include/utils/memutils_memorychunk.h
index 2eefc138e3..38702efc58 100644
--- a/src/include/utils/memutils_memorychunk.h
+++ b/src/include/utils/memutils_memorychunk.h
@@ -156,7 +156,7 @@ MemoryChunkSetHdrMask(MemoryChunk *chunk, void *block,
{
Size blockoffset = (char *) chunk - (char *) block;
- Assert((char *) chunk > (char *) block);
+ Assert((char *) chunk >= (char *) block);
Assert(blockoffset <= MEMORYCHUNK_MAX_BLOCKOFFSET);
Assert(value <= MEMORYCHUNK_MAX_VALUE);
Assert((int) methodid <= MEMORY_CONTEXT_METHODID_MASK);
diff --git a/src/include/utils/palloc.h b/src/include/utils/palloc.h
index 8eee0e2938..989ddf18ef 100644
--- a/src/include/utils/palloc.h
+++ b/src/include/utils/palloc.h
@@ -73,10 +73,13 @@ extern void *MemoryContextAllocZero(MemoryContext context,
Size size);
extern void *MemoryContextAllocZeroAligned(MemoryContext context, Size size);
extern void *MemoryContextAllocExtended(MemoryContext context,
Size size, int flags);
+extern void *MemoryContextAllocAligned(MemoryContext context,
+ Size
size, Size alignto, int flags);
extern void *palloc(Size size);
extern void *palloc0(Size size);
extern void *palloc_extended(Size size, int flags);
+extern void *palloc_aligned(Size size, Size alignto, int flags);
extern pg_nodiscard void *repalloc(void *pointer, Size size);
extern pg_nodiscard void *repalloc_extended(void *pointer,
Size size, int flags);
