Back-patch fix to avoid integer overflow in ExecHashJoinGetBatch(),

which leads to core dump in large-enough hash joins.

diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c
index bf9bf6eeb0f5120318eeccf024714312ed043235..ed46f544202d05b9fe14bd4661001f8d3967dbe8 100644 (file)
--- a/src/backend/executor/nodeHash.c
+++ b/src/backend/executor/nodeHash.c
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  *
- *     $Id: nodeHash.c,v 1.60 2001/10/25 05:49:28 momjian Exp $
+ *     $Id: nodeHash.c,v 1.60.2.1 2003/01/29 19:37:23 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -19,6 +19,7 @@
  */
 #include "postgres.h"
 
+#include <limits.h>
 #include <sys/types.h>
 #include <math.h>
 
@@ -343,7 +344,8 @@ ExecChooseHashTableSize(double ntuples, int tupwidth,
 {
        int                     tupsize;
        double          inner_rel_bytes;
-       double          hash_table_bytes;
+       long            hash_table_bytes;
+       double          dtmp;
        int                     nbatch;
        int                     nbuckets;
        int                     totalbuckets;
@@ -361,20 +363,22 @@ ExecChooseHashTableSize(double ntuples, int tupwidth,
        inner_rel_bytes = ntuples * tupsize * FUDGE_FAC;
 
        /*
-        * Target hashtable size is SortMem kilobytes, but not less than
-        * sqrt(estimated inner rel size), so as to avoid horrible
-        * performance.
+        * Target in-memory hashtable size is SortMem kilobytes.
         */
-       hash_table_bytes = sqrt(inner_rel_bytes);
-       if (hash_table_bytes < (SortMem * 1024L))
-               hash_table_bytes = SortMem * 1024L;
+       hash_table_bytes = SortMem * 1024L;
 
        /*
         * Count the number of hash buckets we want for the whole relation,
         * for an average bucket load of NTUP_PER_BUCKET (per virtual
-        * bucket!).
+        * bucket!).  It has to fit in an int, however.
         */
-       totalbuckets = (int) ceil(ntuples * FUDGE_FAC / NTUP_PER_BUCKET);
+       dtmp = ceil(ntuples * FUDGE_FAC / NTUP_PER_BUCKET);
+       if (dtmp < INT_MAX)
+               totalbuckets = (int) dtmp;
+       else
+               totalbuckets = INT_MAX;
+       if (totalbuckets <= 0)
+               totalbuckets = 1;
 
        /*
         * Count the number of buckets we think will actually fit in the
@@ -408,10 +412,16 @@ ExecChooseHashTableSize(double ntuples, int tupwidth,
                 * that nbatch doesn't have to have anything to do with the ratio
                 * totalbuckets/nbuckets; in fact, it is the number of groups we
                 * will use for the part of the data that doesn't fall into the
-                * first nbuckets hash buckets.
+                * first nbuckets hash buckets.  We try to set it to make all the
+                * batches the same size.  But we have to keep nbatch small
+                * enough to avoid integer overflow in ExecHashJoinGetBatch().
                 */
-               nbatch = (int) ceil((inner_rel_bytes - hash_table_bytes) /
-                                                       hash_table_bytes);
+               dtmp = ceil((inner_rel_bytes - hash_table_bytes) /
+                                       hash_table_bytes);
+               if (dtmp < INT_MAX / totalbuckets)
+                       nbatch = (int) dtmp;
+               else
+                       nbatch = INT_MAX / totalbuckets;
                if (nbatch <= 0)
                        nbatch = 1;
        }