Change logtape/tuplestore code to use int64 for block numbers

The code previously relied on "long" as type to track block numbers,
which would be 4 bytes in all Windows builds or any 32-bit builds.  This
limited the code to be able to handle up to 16TB of data with the
default block size of 8kB, like during a CLUSTER.  This code now relies
on a more portable int64, which should be more than enough for at least
the next 20 years to come.

This issue has been reported back in 2017, but nothing was done about it
back then, so here we go now.

Reported-by: Peter Geoghegan
Reviewed-by: Heikki Linnakangas
Discussion: https://postgr.es/m/CAH2-WznCscXnWmnj=STC0aSa7QG+BRedDnZsP=Jo_R9GUZvUrg@mail.gmail.com

diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c
index d3f0f58858021cbae0308b851472b0e4ef3649ae..2ca37832331c49d5fcd2c8b4a77b0f03db0e26c9 100644 (file)
--- a/src/backend/storage/file/buffile.c
+++ b/src/backend/storage/file/buffile.c
@@ -841,14 +841,14 @@ BufFileTell(BufFile *file, int *fileno, off_t *offset)
  *
  * Performs absolute seek to the start of the n'th BLCKSZ-sized block of
  * the file.  Note that users of this interface will fail if their files
- * exceed BLCKSZ * LONG_MAX bytes, but that is quite a lot; we don't work
- * with tables bigger than that, either...
+ * exceed BLCKSZ * PG_INT64_MAX bytes, but that is quite a lot; we don't
+ * work with tables bigger than that, either...
  *
  * Result is 0 if OK, EOF if not.  Logical position is not moved if an
  * impossible seek is attempted.
  */
 int
-BufFileSeekBlock(BufFile *file, long blknum)
+BufFileSeekBlock(BufFile *file, int64 blknum)
 {
        return BufFileSeek(file,
                                           (int) (blknum / BUFFILE_SEG_SIZE),
@@ -901,10 +901,10 @@ BufFileSize(BufFile *file)
  * begins.  Caller should apply this as an offset when working off block
  * positions that are in terms of the original BufFile space.
  */
-long
+int64
 BufFileAppend(BufFile *target, BufFile *source)
 {
-       long            startBlock = target->numFiles * BUFFILE_SEG_SIZE;
+       int64           startBlock = target->numFiles * BUFFILE_SEG_SIZE;
        int                     newNumFiles = target->numFiles + source->numFiles;
        int                     i;
 

diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index f31878bdee1aec441bd1f80e10266eed1907c9c9..604fd003081f7388cb82d8952481618626385606 100644 (file)
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -94,9 +94,9 @@
  */
 typedef struct TapeBlockTrailer
 {
-       long            prev;                   /* previous block on this tape, or -1 on first
+       int64           prev;                   /* previous block on this tape, or -1 on first
                                                                 * block */
-       long            next;                   /* next block on this tape, or # of valid
+       int64           next;                   /* next block on this tape, or # of valid
                                                                 * bytes on last block (if < 0) */
 } TapeBlockTrailer;
 
@@ -153,10 +153,10 @@ struct LogicalTape
         * When concatenation of worker tape BufFiles is performed, an offset to
         * the first block in the unified BufFile space is applied during reads.
         */
-       long            firstBlockNumber;
-       long            curBlockNumber;
-       long            nextBlockNumber;
-       long            offsetBlockNumber;
+       int64           firstBlockNumber;
+       int64           curBlockNumber;
+       int64           nextBlockNumber;
+       int64           offsetBlockNumber;
 
        /*
         * Buffer for current data block(s).
@@ -172,7 +172,7 @@ struct LogicalTape
         * order; blocks are consumed from the end of the array (lowest block
         * numbers first).
         */
-       long       *prealloc;
+       int64      *prealloc;
        int                     nprealloc;              /* number of elements in list */
        int                     prealloc_size;  /* number of elements list can hold */
 };
@@ -200,9 +200,9 @@ struct LogicalTapeSet
         * blocks that are in unused holes between worker spaces following BufFile
         * concatenation.
         */
-       long            nBlocksAllocated;       /* # of blocks allocated */
-       long            nBlocksWritten; /* # of blocks used in underlying file */
-       long            nHoleBlocks;    /* # of "hole" blocks left */
+       int64           nBlocksAllocated;       /* # of blocks allocated */
+       int64           nBlocksWritten; /* # of blocks used in underlying file */
+       int64           nHoleBlocks;    /* # of "hole" blocks left */
 
        /*
         * We store the numbers of recycled-and-available blocks in freeBlocks[].
@@ -213,19 +213,19 @@ struct LogicalTapeSet
         * LogicalTapeSetForgetFreeSpace().
         */
        bool            forgetFreeSpace;        /* are we remembering free blocks? */
-       long       *freeBlocks;         /* resizable array holding minheap */
-       long            nFreeBlocks;    /* # of currently free blocks */
+       int64      *freeBlocks;         /* resizable array holding minheap */
+       int64           nFreeBlocks;    /* # of currently free blocks */
        Size            freeBlocksLen;  /* current allocated length of freeBlocks[] */
        bool            enable_prealloc;        /* preallocate write blocks? */
 };
 
 static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
-static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, const void *buffer);
-static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
-static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
-static long ltsGetFreeBlock(LogicalTapeSet *lts);
-static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
-static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
+static void ltsWriteBlock(LogicalTapeSet *lts, int64 blocknum, const void *buffer);
+static void ltsReadBlock(LogicalTapeSet *lts, int64 blocknum, void *buffer);
+static int64 ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
+static int64 ltsGetFreeBlock(LogicalTapeSet *lts);
+static int64 ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsReleaseBlock(LogicalTapeSet *lts, int64 blocknum);
 static void ltsInitReadBuffer(LogicalTape *lt);
 
 
@@ -235,7 +235,7 @@ static void ltsInitReadBuffer(LogicalTape *lt);
  * No need for an error return convention; we ereport() on any error.
  */
 static void
-ltsWriteBlock(LogicalTapeSet *lts, long blocknum, const void *buffer)
+ltsWriteBlock(LogicalTapeSet *lts, int64 blocknum, const void *buffer)
 {
        /*
         * BufFile does not support "holes", so if we're about to write a block
@@ -263,8 +263,8 @@ ltsWriteBlock(LogicalTapeSet *lts, long blocknum, const void *buffer)
        if (BufFileSeekBlock(lts->pfile, blocknum) != 0)
                ereport(ERROR,
                                (errcode_for_file_access(),
-                                errmsg("could not seek to block %ld of temporary file",
-                                               blocknum)));
+                                errmsg("could not seek to block %lld of temporary file",
+                                               (long long) blocknum)));
        BufFileWrite(lts->pfile, buffer, BLCKSZ);
 
        /* Update nBlocksWritten, if we extended the file */
@@ -279,13 +279,13 @@ ltsWriteBlock(LogicalTapeSet *lts, long blocknum, const void *buffer)
  * module should never attempt to read a block it doesn't know is there.
  */
 static void
-ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
+ltsReadBlock(LogicalTapeSet *lts, int64 blocknum, void *buffer)
 {
        if (BufFileSeekBlock(lts->pfile, blocknum) != 0)
                ereport(ERROR,
                                (errcode_for_file_access(),
-                                errmsg("could not seek to block %ld of temporary file",
-                                               blocknum)));
+                                errmsg("could not seek to block %lld of temporary file",
+                                               (long long) blocknum)));
        BufFileReadExact(lts->pfile, buffer, BLCKSZ);
 }
 
@@ -303,7 +303,7 @@ ltsReadFillBuffer(LogicalTape *lt)
        do
        {
                char       *thisbuf = lt->buffer + lt->nbytes;
-               long            datablocknum = lt->nextBlockNumber;
+               int64           datablocknum = lt->nextBlockNumber;
 
                /* Fetch next block number */
                if (datablocknum == -1L)
@@ -333,20 +333,20 @@ ltsReadFillBuffer(LogicalTape *lt)
        return (lt->nbytes > 0);
 }
 
-static inline unsigned long
-left_offset(unsigned long i)
+static inline uint64
+left_offset(uint64 i)
 {
        return 2 * i + 1;
 }
 
-static inline unsigned long
-right_offset(unsigned long i)
+static inline uint64
+right_offset(uint64 i)
 {
        return 2 * i + 2;
 }
 
-static inline unsigned long
-parent_offset(unsigned long i)
+static inline uint64
+parent_offset(uint64 i)
 {
        return (i - 1) / 2;
 }
@@ -354,7 +354,7 @@ parent_offset(unsigned long i)
 /*
  * Get the next block for writing.
  */
-static long
+static int64
 ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt)
 {
        if (lts->enable_prealloc)
@@ -367,14 +367,14 @@ ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt)
  * Select the lowest currently unused block from the tape set's global free
  * list min heap.
  */
-static long
+static int64
 ltsGetFreeBlock(LogicalTapeSet *lts)
 {
-       long       *heap = lts->freeBlocks;
-       long            blocknum;
-       long            heapsize;
-       long            holeval;
-       unsigned long holepos;
+       int64      *heap = lts->freeBlocks;
+       int64           blocknum;
+       int64           heapsize;
+       int64           holeval;
+       uint64          holepos;
 
        /* freelist empty; allocate a new block */
        if (lts->nFreeBlocks == 0)
@@ -398,9 +398,9 @@ ltsGetFreeBlock(LogicalTapeSet *lts)
        heapsize = lts->nFreeBlocks;
        while (true)
        {
-               unsigned long left = left_offset(holepos);
-               unsigned long right = right_offset(holepos);
-               unsigned long min_child;
+               uint64          left = left_offset(holepos);
+               uint64          right = right_offset(holepos);
+               uint64          min_child;
 
                if (left < heapsize && right < heapsize)
                        min_child = (heap[left] < heap[right]) ? left : right;
@@ -427,7 +427,7 @@ ltsGetFreeBlock(LogicalTapeSet *lts)
  * Refill the preallocation list with blocks from the tape set's free list if
  * necessary.
  */
-static long
+static int64
 ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt)
 {
        /* sorted in descending order, so return the last element */
@@ -437,7 +437,7 @@ ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt)
        if (lt->prealloc == NULL)
        {
                lt->prealloc_size = TAPE_WRITE_PREALLOC_MIN;
-               lt->prealloc = (long *) palloc(sizeof(long) * lt->prealloc_size);
+               lt->prealloc = (int64 *) palloc(sizeof(int64) * lt->prealloc_size);
        }
        else if (lt->prealloc_size < TAPE_WRITE_PREALLOC_MAX)
        {
@@ -445,8 +445,8 @@ ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt)
                lt->prealloc_size *= 2;
                if (lt->prealloc_size > TAPE_WRITE_PREALLOC_MAX)
                        lt->prealloc_size = TAPE_WRITE_PREALLOC_MAX;
-               lt->prealloc = (long *) repalloc(lt->prealloc,
-                                                                                sizeof(long) * lt->prealloc_size);
+               lt->prealloc = (int64 *) repalloc(lt->prealloc,
+                                                                                 sizeof(int64) * lt->prealloc_size);
        }
 
        /* refill preallocation list */
@@ -466,10 +466,10 @@ ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt)
  * Return a block# to the freelist.
  */
 static void
-ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
+ltsReleaseBlock(LogicalTapeSet *lts, int64 blocknum)
 {
-       long       *heap;
-       unsigned long holepos;
+       int64      *heap;
+       uint64          holepos;
 
        /*
         * Do nothing if we're no longer interested in remembering free space.
@@ -486,12 +486,12 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
                 * If the freelist becomes very large, just return and leak this free
                 * block.
                 */
-               if (lts->freeBlocksLen * 2 * sizeof(long) > MaxAllocSize)
+               if (lts->freeBlocksLen * 2 * sizeof(int64) > MaxAllocSize)
                        return;
 
                lts->freeBlocksLen *= 2;
-               lts->freeBlocks = (long *) repalloc(lts->freeBlocks,
-                                                                                       lts->freeBlocksLen * sizeof(long));
+               lts->freeBlocks = (int64 *) repalloc(lts->freeBlocks,
+                                                                                        lts->freeBlocksLen * sizeof(int64));
        }
 
        /* create a "hole" at end of minheap array */
@@ -502,7 +502,7 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
        /* sift up to insert blocknum */
        while (holepos != 0)
        {
-               unsigned long parent = parent_offset(holepos);
+               uint64          parent = parent_offset(holepos);
 
                if (heap[parent] < blocknum)
                        break;
@@ -566,7 +566,7 @@ LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
        lts->nHoleBlocks = 0L;
        lts->forgetFreeSpace = false;
        lts->freeBlocksLen = 32;        /* reasonable initial guess */
-       lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
+       lts->freeBlocks = (int64 *) palloc(lts->freeBlocksLen * sizeof(int64));
        lts->nFreeBlocks = 0;
        lts->enable_prealloc = preallocate;
 
@@ -609,7 +609,7 @@ LogicalTape *
 LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
 {
        LogicalTape *lt;
-       long            tapeblocks;
+       int64           tapeblocks;
        char            filename[MAXPGPATH];
        BufFile    *file;
        int64           filesize;
@@ -789,7 +789,7 @@ LogicalTapeWrite(LogicalTape *lt, const void *ptr, size_t size)
                if (lt->pos >= (int) TapeBlockPayloadSize)
                {
                        /* Buffer full, dump it out */
-                       long            nextBlockNumber;
+                       int64           nextBlockNumber;
 
                        if (!lt->dirty)
                        {
@@ -1086,7 +1086,7 @@ LogicalTapeBackspace(LogicalTape *lt, size_t size)
        seekpos = (size_t) lt->pos; /* part within this block */
        while (size > seekpos)
        {
-               long            prev = TapeBlockGetTrailer(lt->buffer)->prev;
+               int64           prev = TapeBlockGetTrailer(lt->buffer)->prev;
 
                if (prev == -1L)
                {
@@ -1100,10 +1100,10 @@ LogicalTapeBackspace(LogicalTape *lt, size_t size)
                ltsReadBlock(lt->tapeSet, prev, lt->buffer);
 
                if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
-                       elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
-                                prev,
-                                TapeBlockGetTrailer(lt->buffer)->next,
-                                lt->curBlockNumber);
+                       elog(ERROR, "broken tape, next of block %lld is %lld, expected %lld",
+                                (long long) prev,
+                                (long long) (TapeBlockGetTrailer(lt->buffer)->next),
+                                (long long) lt->curBlockNumber);
 
                lt->nbytes = TapeBlockPayloadSize;
                lt->curBlockNumber = prev;
@@ -1130,7 +1130,7 @@ LogicalTapeBackspace(LogicalTape *lt, size_t size)
  * LogicalTapeTell().
  */
 void
-LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, int64 blocknum, int offset)
 {
        Assert(lt->frozen);
        Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
@@ -1159,7 +1159,7 @@ LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
  * the position for a seek after freezing.  Not clear if anyone needs that.
  */
 void
-LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, int64 *blocknum, int *offset)
 {
        if (lt->buffer == NULL)
                ltsInitReadBuffer(lt);
@@ -1179,7 +1179,7 @@ LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
  * This should not be called while there are open write buffers; otherwise it
  * may not account for buffered data.
  */
-long
+int64
 LogicalTapeSetBlocks(LogicalTapeSet *lts)
 {
        return lts->nBlocksWritten - lts->nHoleBlocks;

diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index ab6353bdcd1c42cc5050a3cec6ae0c4b65a9cfcd..b38022356bb8c47b8c52928bc55a8fbce73a3726 100644 (file)
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -296,7 +296,7 @@ struct Tuplesortstate
        bool            eof_reached;    /* reached EOF (needed for cursors) */
 
        /* markpos_xxx holds marked position for mark and restore */
-       long            markpos_block;  /* tape block# (only used if SORTEDONTAPE) */
+       int64           markpos_block;  /* tape block# (only used if SORTEDONTAPE) */
        int                     markpos_offset; /* saved "current", or offset in tape block */
        bool            markpos_eof;    /* saved "eof_reached" */
 
@@ -903,7 +903,7 @@ tuplesort_free(Tuplesortstate *state)
        MemoryContext oldcontext = MemoryContextSwitchTo(state->base.sortcontext);
 
 #ifdef TRACE_SORT
-       long            spaceUsed;
+       int64           spaceUsed;
 
        if (state->tapeset)
                spaceUsed = LogicalTapeSetBlocks(state->tapeset);
@@ -928,13 +928,13 @@ tuplesort_free(Tuplesortstate *state)
        if (trace_sort)
        {
                if (state->tapeset)
-                       elog(LOG, "%s of worker %d ended, %ld disk blocks used: %s",
+                       elog(LOG, "%s of worker %d ended, %lld disk blocks used: %s",
                                 SERIAL(state) ? "external sort" : "parallel external sort",
-                                state->worker, spaceUsed, pg_rusage_show(&state->ru_start));
+                                state->worker, (long long) spaceUsed, pg_rusage_show(&state->ru_start));
                else
-                       elog(LOG, "%s of worker %d ended, %ld KB used: %s",
+                       elog(LOG, "%s of worker %d ended, %lld KB used: %s",
                                 SERIAL(state) ? "internal sort" : "unperformed parallel sort",
-                                state->worker, spaceUsed, pg_rusage_show(&state->ru_start));
+                                state->worker, (long long) spaceUsed, pg_rusage_show(&state->ru_start));
        }
 
        TRACE_POSTGRESQL_SORT_DONE(state->tapeset != NULL, spaceUsed);

diff --git a/src/include/storage/buffile.h b/src/include/storage/buffile.h
index 658376671933c7295d0814a717d5b6a0440d85ba..cbffc9c77e3a104c353f2a8c10815b34f5ca7e48 100644 (file)
--- a/src/include/storage/buffile.h
+++ b/src/include/storage/buffile.h
@@ -44,9 +44,9 @@ extern size_t BufFileReadMaybeEOF(BufFile *file, void *ptr, size_t size, bool eo
 extern void BufFileWrite(BufFile *file, const void *ptr, size_t size);
 extern int     BufFileSeek(BufFile *file, int fileno, off_t offset, int whence);
 extern void BufFileTell(BufFile *file, int *fileno, off_t *offset);
-extern int     BufFileSeekBlock(BufFile *file, long blknum);
+extern int     BufFileSeekBlock(BufFile *file, int64 blknum);
 extern int64 BufFileSize(BufFile *file);
-extern long BufFileAppend(BufFile *target, BufFile *source);
+extern int64 BufFileAppend(BufFile *target, BufFile *source);
 
 extern BufFile *BufFileCreateFileSet(FileSet *fileset, const char *name);
 extern void BufFileExportFileSet(BufFile *file);

diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 5420a24ac9b9a3c2e2bfd860587e6b82318da9cb..2de7add81c5ec0abde8e6aa70a7a48fa109d7860 100644 (file)
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -51,7 +51,7 @@ typedef struct TapeShare
         * Currently, all the leader process needs is the location of the
         * materialized tape's first block.
         */
-       long            firstblocknumber;
+       int64           firstblocknumber;
 } TapeShare;
 
 /*
@@ -70,8 +70,8 @@ extern void LogicalTapeWrite(LogicalTape *lt, const void *ptr, size_t size);
 extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
 extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
 extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
-extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
-extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
+extern void LogicalTapeSeek(LogicalTape *lt, int64 blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, int64 *blocknum, int *offset);
+extern int64 LogicalTapeSetBlocks(LogicalTapeSet *lts);
 
 #endif                                                 /* LOGTAPE_H */