Add allocator support for larger allocation alignment & use for IO.

Author:
Reviewed-By:
Discussion: https://postgr.es/m/
Backpatch:

diff --git a/contrib/bloom/blinsert.c b/contrib/bloom/blinsert.c
index 32b5d62e1f34cf9bdc46a142beee52fa83182b46..42e69b53b840dee7edb6cc4a180d86d2dbd7eb42 100644 (file)
--- a/contrib/bloom/blinsert.c
+++ b/contrib/bloom/blinsert.c
@@ -167,7 +167,7 @@ blbuildempty(Relation index)
    Page        metapage;
 
    /* Construct metapage. */
-   metapage = (Page) palloc(BLCKSZ);
+   metapage = (Page) palloc_io_aligned(BLCKSZ, 0);
    BloomFillMetapage(index, metapage);
 
    /*

diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c
index 1054f6f1f2e34c996c86052fab10896542ee56df..7cce8f6d5a6a22f2646b0298c0be80fab4398d7b 100644 (file)
--- a/src/backend/access/gist/gistbuild.c
+++ b/src/backend/access/gist/gistbuild.c
@@ -408,7 +408,7 @@ gist_indexsortbuild(GISTBuildState *state)
     * Write an empty page as a placeholder for the root page. It will be
     * replaced with the real root page at the end.
     */
-   page = palloc0(BLCKSZ);
+   page = palloc_io_aligned(BLCKSZ, MCXT_ALLOC_ZERO);
    RelationOpenSmgr(state->indexrel);
    smgrextend(state->indexrel->rd_smgr, MAIN_FORKNUM, GIST_ROOT_BLKNO,
               page, true);
@@ -530,7 +530,7 @@ gist_indexsortbuild_pagestate_flush(GISTBuildState *state,
    if (parent == NULL)
    {
        parent = palloc(sizeof(GistSortedBuildPageState));
-       parent->page = (Page) palloc(BLCKSZ);
+       parent->page = (Page) palloc_io_aligned(BLCKSZ, 0);
        parent->parent = NULL;
        gistinitpage(parent->page, 0);
 
@@ -539,7 +539,7 @@ gist_indexsortbuild_pagestate_flush(GISTBuildState *state,
    gist_indexsortbuild_pagestate_add(state, parent, union_tuple);
 
    /* Re-initialize the page buffer for next page on this level. */
-   pagestate->page = palloc(BLCKSZ);
+   pagestate->page = palloc_io_aligned(BLCKSZ, 0);
    gistinitpage(pagestate->page, isleaf ? F_LEAF : 0);
 
    /*

diff --git a/src/backend/access/gist/gistbuildbuffers.c b/src/backend/access/gist/gistbuildbuffers.c
index 95cc3348442b9323bb6336863c1df55832afb893..c3a06b1f0ab555b18eecf0956570f8c435fe2bac 100644 (file)
--- a/src/backend/access/gist/gistbuildbuffers.c
+++ b/src/backend/access/gist/gistbuildbuffers.c
@@ -186,8 +186,9 @@ gistAllocateNewPageBuffer(GISTBuildBuffers *gfbb)
 {
    GISTNodeBufferPage *pageBuffer;
 
-   pageBuffer = (GISTNodeBufferPage *) MemoryContextAllocZero(gfbb->context,
-                                                              BLCKSZ);
+   pageBuffer = (GISTNodeBufferPage *)
+       MemoryContextAllocIOAligned(gfbb->context,
+                                   BLCKSZ, MCXT_ALLOC_ZERO);
    pageBuffer->prev = InvalidBlockNumber;
 
    /* Set page free space */

diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c
index 29ffe4067042c1d9b2290e0534af77d01c7fe81f..ac8b4e101bcfa24dae8261e6b5797fcf7fd99d1d 100644 (file)
--- a/src/backend/access/heap/rewriteheap.c
+++ b/src/backend/access/heap/rewriteheap.c
@@ -256,7 +256,7 @@ begin_heap_rewrite(Relation old_heap, Relation new_heap, TransactionId oldest_xm
 
    state->rs_old_rel = old_heap;
    state->rs_new_rel = new_heap;
-   state->rs_buffer = (Page) palloc(BLCKSZ);
+   state->rs_buffer = (Page) palloc_io_aligned(BLCKSZ, 0);
    /* new_heap needn't be empty, just locked */
    state->rs_blockno = RelationGetNumberOfBlocks(new_heap);
    state->rs_buffer_valid = false;

diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index ba79a7f3e9e2c4e59089a7e5e1d9569fb6de9c39..0e83b7f333789d0de728c31cd24a94844d47cac7 100644 (file)
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -165,7 +165,7 @@ btbuildempty(Relation index)
    Page        metapage;
 
    /* Construct metapage. */
-   metapage = (Page) palloc(BLCKSZ);
+   metapage = (Page) palloc_io_aligned(BLCKSZ, 0);
    _bt_initmetapage(metapage, P_NONE, 0, _bt_allequalimage(index, false));
 
    /*

diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c
index c904fc8ef7bd3e27752ce59f1dd74774f3f2581c..9615d0cc6d0dccd7a11369a241139a2cb06ac666 100644 (file)
--- a/src/backend/access/nbtree/nbtsort.c
+++ b/src/backend/access/nbtree/nbtsort.c
@@ -613,7 +613,7 @@ _bt_blnewpage(uint32 level)
    Page        page;
    BTPageOpaque opaque;
 
-   page = (Page) palloc(BLCKSZ);
+   page = (Page) palloc_io_aligned(BLCKSZ, 0);
 
    /* Zero the page and set up standard page header info */
    _bt_pageinit(page, BLCKSZ);
@@ -657,7 +657,9 @@ _bt_blwritepage(BTWriteState *wstate, Page page, BlockNumber blkno)
    while (blkno > wstate->btws_pages_written)
    {
        if (!wstate->btws_zeropage)
-           wstate->btws_zeropage = (Page) palloc0(BLCKSZ);
+           wstate->btws_zeropage =
+               (Page) palloc_io_aligned(BLCKSZ, MCXT_ALLOC_ZERO);
+
        /* don't set checksum for all-zero page */
        smgrextend(wstate->index->rd_smgr, MAIN_FORKNUM,
                   wstate->btws_pages_written++,
@@ -1167,7 +1169,7 @@ _bt_uppershutdown(BTWriteState *wstate, BTPageState *state)
     * set to point to "P_NONE").  This changes the index to the "valid" state
     * by filling in a valid magic number in the metapage.
     */
-   metapage = (Page) palloc(BLCKSZ);
+   metapage = (Page) palloc_io_aligned(BLCKSZ, 0);
    _bt_initmetapage(metapage, rootblkno, rootlevel,
                     wstate->inskey->allequalimage);
    _bt_blwritepage(wstate, metapage, BTREE_METAPAGE);

diff --git a/src/backend/access/spgist/spginsert.c b/src/backend/access/spgist/spginsert.c
index 2e1d8a33d1f7d07b2dc697326e24baaa067dcbfe..1a44bcd205f291e7b087080f8ea806d366e8b915 100644 (file)
--- a/src/backend/access/spgist/spginsert.c
+++ b/src/backend/access/spgist/spginsert.c
@@ -158,7 +158,7 @@ spgbuildempty(Relation index)
    Page        page;
 
    /* Construct metapage. */
-   page = (Page) palloc(BLCKSZ);
+   page = (Page) palloc_io_aligned(BLCKSZ, 0);
    SpGistInitMetapage(page);
 
    /*

diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c
index 850ef8af4f1ef4a571485c14c24342d69f85e681..d611dac26b87ea3bf02bc89b586b2ccf577c0565 100644 (file)
--- a/src/backend/storage/buffer/buf_init.c
+++ b/src/backend/storage/buffer/buf_init.c
@@ -78,8 +78,9 @@ InitBufferPool(void)
                        &foundDescs);
 
    BufferBlocks = (char *)
-       ShmemInitStruct("Buffer Blocks",
-                       NBuffers * (Size) BLCKSZ, &foundBufs);
+       TYPEALIGN(BLCKSZ,
+                 ShmemInitStruct("Buffer Blocks",
+                                 (NBuffers + 1) * (Size) BLCKSZ, &foundBufs));
 
    /* Align lwlocks to cacheline boundary */
    BufferIOCVArray = (ConditionVariableMinimallyPadded *)
@@ -163,6 +164,8 @@ BufferShmemSize(void)
    size = add_size(size, PG_CACHE_LINE_SIZE);
 
    /* size of data pages */
+    /* to allow aligning buffer blocks */
+    size = add_size(size, BLCKSZ);
    size = add_size(size, mul_size(NBuffers, BLCKSZ));
 
    /* size of stuff controlled by freelist.c */

diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c
index 04b3558ea3350a79a1a03b3b1061b040952e8964..eddd0782f298e6bbe486a6508196627b7b8d19e6 100644 (file)
--- a/src/backend/storage/buffer/localbuf.c
+++ b/src/backend/storage/buffer/localbuf.c
@@ -525,8 +525,8 @@ GetLocalBufferStorage(void)
        /* And don't overflow MaxAllocSize, either */
        num_bufs = Min(num_bufs, MaxAllocSize / BLCKSZ);
 
-       cur_block = (char *) MemoryContextAlloc(LocalBufferContext,
-                                               num_bufs * BLCKSZ);
+       cur_block = (char *) MemoryContextAllocIOAligned(LocalBufferContext,
+                                                        num_bufs * BLCKSZ, 0);
        next_buf_in_block = 0;
        num_bufs_in_block = num_bufs;
    }

diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c
index 9ac556b4ae0a731c26806bf039790eafe02dc405..93cdd74449845f0a3f7ef5d9404dd675c6b0cfc8 100644 (file)
--- a/src/backend/storage/page/bufpage.c
+++ b/src/backend/storage/page/bufpage.c
@@ -1405,7 +1405,7 @@ PageSetChecksumCopy(Page page, BlockNumber blkno)
     * and second to avoid wasting space in processes that never call this.
     */
    if (pageCopy == NULL)
-       pageCopy = MemoryContextAlloc(TopMemoryContext, BLCKSZ);
+       pageCopy = MemoryContextAllocIOAligned(TopMemoryContext, BLCKSZ, 0);
 
    memcpy(pageCopy, (char *) page, BLCKSZ);
    ((PageHeader) pageCopy)->pd_checksum = pg_checksum_page(pageCopy, blkno);

diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c
index 0643d714fb1c5e46cebe9e87f4568b524a34b81b..aa66590cf3f2ba349fac3e5ea6b783cf4cdc8bad 100644 (file)
--- a/src/backend/storage/smgr/md.c
+++ b/src/backend/storage/smgr/md.c
@@ -418,6 +418,8 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
    int         nbytes;
    MdfdVec    *v;
 
+   AssertPointerAlignment(buffer, 4096);
+
    /* This assert is too expensive to have on normally ... */
 #ifdef CHECK_WRITE_VS_EXTEND
    Assert(blocknum >= mdnblocks(reln, forknum));
@@ -636,6 +638,8 @@ mdread(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
    int         nbytes;
    MdfdVec    *v;
 
+   AssertPointerAlignment(buffer, 4096);
+
    TRACE_POSTGRESQL_SMGR_MD_READ_START(forknum, blocknum,
                                        reln->smgr_rnode.node.spcNode,
                                        reln->smgr_rnode.node.dbNode,
@@ -701,6 +705,8 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
    int         nbytes;
    MdfdVec    *v;
 
+   AssertPointerAlignment(buffer, 4096);
+
    /* This assert is too expensive to have on normally ... */
 #ifdef CHECK_WRITE_VS_EXTEND
    Assert(blocknum < mdnblocks(reln, forknum));
@@ -1250,7 +1256,7 @@ _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno,
             */
            if (nblocks < ((BlockNumber) RELSEG_SIZE))
            {
-               char       *zerobuf = palloc0(BLCKSZ);
+               char       *zerobuf = palloc_io_aligned(BLCKSZ, MCXT_ALLOC_ZERO);
 
                mdextend(reln, forknum,
                         nextsegno * ((BlockNumber) RELSEG_SIZE) - 1,

diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c
index 84472b9158e7be5521475facc05c07c9a34d6a15..ec811d0a50544e2864017ee07d6da20eba96adb9 100644 (file)
--- a/src/backend/utils/mmgr/mcxt.c
+++ b/src/backend/utils/mmgr/mcxt.c
@@ -59,6 +59,18 @@ static void MemoryContextStatsInternal(MemoryContext context, int level,
 static void MemoryContextStatsPrint(MemoryContext context, void *passthru,
                                    const char *stats_string);
 
+static void AlignedAllocFree(MemoryContext context, void *pointer);
+
+
+static const MemoryContextMethods AlignedRedirectMethods = {
+   .free_p = AlignedAllocFree,
+};
+
+static const MemoryContextData AlignedRedirectContext = {
+   .type = T_AlignedAllocRedirectContext,
+   .methods = &AlignedRedirectMethods
+};
+
 /*
  * You should not do memory allocations within a critical section, because
  * an out-of-memory error will be escalated to a PANIC. To enforce that
@@ -1059,7 +1071,9 @@ pfree(void *pointer)
    MemoryContext context = GetMemoryChunkContext(pointer);
 
    context->methods->free_p(context, pointer);
-   VALGRIND_MEMPOOL_FREE(context, pointer);
+
+   if (context != &AlignedRedirectContext)
+       VALGRIND_MEMPOOL_FREE(context, pointer);
 }
 
 /*
@@ -1221,3 +1235,68 @@ pchomp(const char *in)
        n--;
    return pnstrdup(in, n);
 }
+
+static void
+AlignedAllocFree(MemoryContext context, void *pointer)
+{
+   void *unaligned;
+
+   unaligned = (void *)*(uintptr_t*)((char *) pointer - sizeof(uintptr_t) - sizeof(uintptr_t));
+
+   pfree(unaligned);
+}
+
+void *
+MemoryContextAllocAligned(MemoryContext context,
+                         Size size, Size alignto, int flags)
+{
+   Size        constant_overhead =
+       sizeof(uintptr_t) /* pointer to fake memory context */
+       + sizeof(uintptr_t) /* pointer to actual allocation */
+       ;
+
+   Size        alloc_size;
+   void       *unaligned;
+   void       *aligned;
+
+   /* wouldn't make much sense to waste that much space */
+   AssertArg(alignto < (128 * 1024 * 1024));
+
+   if (alignto < MAXIMUM_ALIGNOF)
+       return palloc_extended(size, flags);
+
+   /* allocate enough space for alignment padding */
+   alloc_size = size + alignto + constant_overhead;
+
+   unaligned = MemoryContextAllocExtended(context, alloc_size, flags);
+
+   aligned = (char *) unaligned + constant_overhead;
+   aligned = (void *) TYPEALIGN(alignto, aligned);
+
+   Assert((uintptr_t)aligned - (uintptr_t)unaligned >= constant_overhead);
+
+   *(uintptr_t*)((char *)aligned - sizeof(uintptr_t)) = (uintptr_t) &AlignedRedirectContext;
+   *(uintptr_t*)((char *)aligned - sizeof(uintptr_t) - sizeof(uintptr_t)) = (uintptr_t) unaligned;
+
+   /* XXX: should we adjust valgrind state here? */
+   return aligned;
+}
+
+void *
+MemoryContextAllocIOAligned(MemoryContext context, Size size, int flags)
+{
+   // FIXME: don't hardcode page size
+   return MemoryContextAllocAligned(context, size, 4096, flags);
+}
+
+void *
+palloc_aligned(Size size, Size alignto, int flags)
+{
+   return MemoryContextAllocAligned(CurrentMemoryContext, size, alignto, flags);
+}
+
+void *
+palloc_io_aligned(Size size, int flags)
+{
+   return MemoryContextAllocIOAligned(CurrentMemoryContext, size, flags);
+}

diff --git a/src/include/nodes/memnodes.h b/src/include/nodes/memnodes.h
index 9331ef80fd9d1f6e64863346fefd6e3f129d8386..132ea18a95198e346c367ec50145199de554e81f 100644 (file)
--- a/src/include/nodes/memnodes.h
+++ b/src/include/nodes/memnodes.h
@@ -99,10 +99,11 @@ typedef struct MemoryContextData
  *
  * Add new context types to the set accepted by this macro.
  */
-#define MemoryContextIsValid(context) \
+#define MemoryContextIsValid(context)                                         \
    ((context) != NULL && \
     (IsA((context), AllocSetContext) || \
      IsA((context), SlabContext) || \
-     IsA((context), GenerationContext)))
+     IsA((context), GenerationContext) || \
+     IsA((context), AlignedAllocRedirectContext)))
 
 #endif                         /* MEMNODES_H */

diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index caed683ba92a8822cb6e100f6611168ac7024f9e..4d3369f8bc915361ad9b7f388feb03002e7788ea 100644 (file)
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -281,6 +281,7 @@ typedef enum NodeTag
    T_AllocSetContext,
    T_SlabContext,
    T_GenerationContext,
+   T_AlignedAllocRedirectContext,
 
    /*
     * TAGS FOR VALUE NODES (value.h)

diff --git a/src/include/utils/palloc.h b/src/include/utils/palloc.h
index 399e5c84a4ad604c04f1422ca3c045def51c6136..642de370e3cd935e472aa0712779cf64593688b0 100644 (file)
--- a/src/include/utils/palloc.h
+++ b/src/include/utils/palloc.h
@@ -73,10 +73,15 @@ extern void *MemoryContextAllocZero(MemoryContext context, Size size);
 extern void *MemoryContextAllocZeroAligned(MemoryContext context, Size size);
 extern void *MemoryContextAllocExtended(MemoryContext context,
                                        Size size, int flags);
+extern void *MemoryContextAllocAligned(MemoryContext context,
+                                      Size size, Size alignto, int flags);
+extern void *MemoryContextAllocIOAligned(MemoryContext context, Size size, int flags);
 
 extern void *palloc(Size size);
 extern void *palloc0(Size size);
 extern void *palloc_extended(Size size, int flags);
+extern void *palloc_aligned(Size size, Size alignto, int flags);
+extern void *palloc_io_aligned(Size size, int flags);
 extern pg_nodiscard void *repalloc(void *pointer, Size size);
 extern void pfree(void *pointer);