Allow FDWs to push down quals without breaking EvalPlanQual rechecks.

This fixes a long-standing bug which was discovered while investigating
the interaction between the new join pushdown code and the EvalPlanQual
machinery: if a ForeignScan appears on the inner side of a paramaterized
nestloop, an EPQ recheck would re-return the original tuple even if
it no longer satisfied the pushed-down quals due to changed parameter
values.

This fix adds a new member to ForeignScan and ForeignScanState and a
new argument to make_foreignscan, and requires changes to FDWs which
push down quals to populate that new argument with a list of quals they
have chosen to push down.  Therefore, I'm only back-patching to 9.5,
even though the bug is not new in 9.5.

Etsuro Fujita, reviewed by me and by Kyotaro Horiguchi.

diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 499f24ff2872e6b6fcecf22033af1bb9f6fa891b..5ce8f90cd8ba333beaa189260c6d7790167fb8f8 100644 (file)
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -563,7 +563,8 @@ fileGetForeignPlan(PlannerInfo *root,
                            scan_relid,
                            NIL,    /* no expressions to evaluate */
                            best_path->fdw_private,
-                           NIL /* no custom tlist */ );
+                           NIL,    /* no custom tlist */
+                           NIL /* no remote quals */ );
 }
 
 /*

diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index e4d799cecd541c777e21e7b48339b83c9e4b17b9..1902f1f4eae1f570a7d4752a0cc678b0399d1931 100644 (file)
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -748,6 +748,7 @@ postgresGetForeignPlan(PlannerInfo *root,
    Index       scan_relid = baserel->relid;
    List       *fdw_private;
    List       *remote_conds = NIL;
+   List       *remote_exprs = NIL;
    List       *local_exprs = NIL;
    List       *params_list = NIL;
    List       *retrieved_attrs;
@@ -769,8 +770,8 @@ postgresGetForeignPlan(PlannerInfo *root,
     *
     * This code must match "extract_actual_clauses(scan_clauses, false)"
     * except for the additional decision about remote versus local execution.
-    * Note however that we only strip the RestrictInfo nodes from the
-    * local_exprs list, since appendWhereClause expects a list of
+    * Note however that we don't strip the RestrictInfo nodes from the
+    * remote_conds list, since appendWhereClause expects a list of
     * RestrictInfos.
     */
    foreach(lc, scan_clauses)
@@ -784,11 +785,17 @@ postgresGetForeignPlan(PlannerInfo *root,
            continue;
 
        if (list_member_ptr(fpinfo->remote_conds, rinfo))
+       {
            remote_conds = lappend(remote_conds, rinfo);
+           remote_exprs = lappend(remote_exprs, rinfo->clause);
+       }
        else if (list_member_ptr(fpinfo->local_conds, rinfo))
            local_exprs = lappend(local_exprs, rinfo->clause);
        else if (is_foreign_expr(root, baserel, rinfo->clause))
+       {
            remote_conds = lappend(remote_conds, rinfo);
+           remote_exprs = lappend(remote_exprs, rinfo->clause);
+       }
        else
            local_exprs = lappend(local_exprs, rinfo->clause);
    }
@@ -874,7 +881,8 @@ postgresGetForeignPlan(PlannerInfo *root,
                            scan_relid,
                            params_list,
                            fdw_private,
-                           NIL /* no custom tlist */ );
+                           NIL,    /* no custom tlist */
+                           remote_exprs);
 }
 
 /*

diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 4c410c791689e4e958985c700047ed3f7d352cfd..1533a6bf80c3f888ab18707d1be6cabf1f337003 100644 (file)
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1135,6 +1135,15 @@ GetForeignServerByName(const char *name, bool missing_ok);
      evaluation of the <structfield>fdw_exprs</> expression tree.
     </para>
 
+    <para>
+     Any clauses removed from the plan node's qual list must instead be added
+     to <literal>fdw_recheck_quals</> in order to ensure correct behavior
+     at the <literal>READ COMMITTED</> isolation level.  When a concurrent
+     update occurs for some other table involved in the query, the executor
+     may need to verify that all of the original quals are still satisfied for
+     the tuple, possibly against a different set of parameter values.
+    </para>
+
     <para>
      Another <structname>ForeignScan</> field that can be filled by FDWs
      is <structfield>fdw_scan_tlist</>, which describes the tuples returned by

diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index bb28a7372d1be4a8dfc4cc9bd378d531af3a745d..6165e4a6cb432b1c32a6ea0104f3d14a68015dd4 100644 (file)
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -25,6 +25,7 @@
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
+#include "utils/memutils.h"
 #include "utils/rel.h"
 
 static TupleTableSlot *ForeignNext(ForeignScanState *node);
@@ -72,8 +73,19 @@ ForeignNext(ForeignScanState *node)
 static bool
 ForeignRecheck(ForeignScanState *node, TupleTableSlot *slot)
 {
-   /* There are no access-method-specific conditions to recheck. */
-   return true;
+   ExprContext *econtext;
+
+   /*
+    * extract necessary information from foreign scan node
+    */
+   econtext = node->ss.ps.ps_ExprContext;
+
+   /* Does the tuple meet the remote qual condition? */
+   econtext->ecxt_scantuple = slot;
+
+   ResetExprContext(econtext);
+
+   return ExecQual(node->fdw_recheck_quals, econtext, false);
 }
 
 /* ----------------------------------------------------------------
@@ -135,6 +147,9 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
    scanstate->ss.ps.qual = (List *)
        ExecInitExpr((Expr *) node->scan.plan.qual,
                     (PlanState *) scanstate);
+   scanstate->fdw_recheck_quals = (List *)
+       ExecInitExpr((Expr *) node->fdw_recheck_quals,
+                    (PlanState *) scanstate);
 
    /*
     * tuple table initialization

diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 0b4ab2310935024259ee87a1cef9455ee3806c19..c176ff978ea306f9542caab8e2618213436fd42b 100644 (file)
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -648,6 +648,7 @@ _copyForeignScan(const ForeignScan *from)
    COPY_NODE_FIELD(fdw_exprs);
    COPY_NODE_FIELD(fdw_private);
    COPY_NODE_FIELD(fdw_scan_tlist);
+   COPY_NODE_FIELD(fdw_recheck_quals);
    COPY_BITMAPSET_FIELD(fs_relids);
    COPY_SCALAR_FIELD(fsSystemCol);
 

diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index df7f6e165aa3b49d39a21f0fc60fb3ec0dd40707..3e75cd1146d4fcdcb6b9311843623b24d91ccd21 100644 (file)
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -594,6 +594,7 @@ _outForeignScan(StringInfo str, const ForeignScan *node)
    WRITE_NODE_FIELD(fdw_exprs);
    WRITE_NODE_FIELD(fdw_private);
    WRITE_NODE_FIELD(fdw_scan_tlist);
+   WRITE_NODE_FIELD(fdw_recheck_quals);
    WRITE_BITMAPSET_FIELD(fs_relids);
    WRITE_BOOL_FIELD(fsSystemCol);
 }

diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 5802a73c7182aefba595592ad6d1c1dcaf7e52be..94ba6dc0b9b1efa7c3286fb052c7dfb05c31fa3e 100644 (file)
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -1798,6 +1798,7 @@ _readForeignScan(void)
    READ_NODE_FIELD(fdw_exprs);
    READ_NODE_FIELD(fdw_private);
    READ_NODE_FIELD(fdw_scan_tlist);
+   READ_NODE_FIELD(fdw_recheck_quals);
    READ_BITMAPSET_FIELD(fs_relids);
    READ_BOOL_FIELD(fsSystemCol);
 

diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index 0ee7392bcce53dc2b80650176f4c3072f4ff1c4a..791b64e7d05a3e36a593f1c6d54dbb317d294996 100644 (file)
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -2153,6 +2153,9 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
            replace_nestloop_params(root, (Node *) scan_plan->scan.plan.qual);
        scan_plan->fdw_exprs = (List *)
            replace_nestloop_params(root, (Node *) scan_plan->fdw_exprs);
+       scan_plan->fdw_recheck_quals = (List *)
+           replace_nestloop_params(root,
+                                   (Node *) scan_plan->fdw_recheck_quals);
    }
 
    /*
@@ -3738,7 +3741,8 @@ make_foreignscan(List *qptlist,
                 Index scanrelid,
                 List *fdw_exprs,
                 List *fdw_private,
-                List *fdw_scan_tlist)
+                List *fdw_scan_tlist,
+                List *fdw_recheck_quals)
 {
    ForeignScan *node = makeNode(ForeignScan);
    Plan       *plan = &node->scan.plan;
@@ -3754,6 +3758,7 @@ make_foreignscan(List *qptlist,
    node->fdw_exprs = fdw_exprs;
    node->fdw_private = fdw_private;
    node->fdw_scan_tlist = fdw_scan_tlist;
+   node->fdw_recheck_quals = fdw_recheck_quals;
    /* fs_relids will be filled in by create_foreignscan_plan */
    node->fs_relids = NULL;
    /* fsSystemCol will be filled in by create_foreignscan_plan */

diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c
index 9392d61dba7c0911f2f0b9aae3a169016a9dffe6..8c6c57101c0baaf8997b15c49ae947c70595bc15 100644 (file)
--- a/src/backend/optimizer/plan/setrefs.c
+++ b/src/backend/optimizer/plan/setrefs.c
@@ -1133,13 +1133,15 @@ set_foreignscan_references(PlannerInfo *root,
    }
    else
    {
-       /* Adjust tlist, qual, fdw_exprs in the standard way */
+       /* Adjust tlist, qual, fdw_exprs, etc. in the standard way */
        fscan->scan.plan.targetlist =
            fix_scan_list(root, fscan->scan.plan.targetlist, rtoffset);
        fscan->scan.plan.qual =
            fix_scan_list(root, fscan->scan.plan.qual, rtoffset);
        fscan->fdw_exprs =
            fix_scan_list(root, fscan->fdw_exprs, rtoffset);
+       fscan->fdw_recheck_quals =
+           fix_scan_list(root, fscan->fdw_recheck_quals, rtoffset);
    }
 
    /* Adjust fs_relids if needed */

diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c
index 6b32f85d6c074c0643aee36f338bf22e9a0995ea..82414d4509c79528651a2d21b85f18b25c586168 100644 (file)
--- a/src/backend/optimizer/plan/subselect.c
+++ b/src/backend/optimizer/plan/subselect.c
@@ -2394,10 +2394,18 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params,
            break;
 
        case T_ForeignScan:
-           finalize_primnode((Node *) ((ForeignScan *) plan)->fdw_exprs,
-                             &context);
-           /* We assume fdw_scan_tlist cannot contain Params */
-           context.paramids = bms_add_members(context.paramids, scan_params);
+           {
+               ForeignScan *fscan = (ForeignScan *) plan;
+
+               finalize_primnode((Node *) fscan->fdw_exprs,
+                                 &context);
+               finalize_primnode((Node *) fscan->fdw_recheck_quals,
+                                 &context);
+
+               /* We assume fdw_scan_tlist cannot contain Params */
+               context.paramids = bms_add_members(context.paramids,
+                                                  scan_params);
+           }
            break;
 
        case T_CustomScan:

diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index b6895f94c397ad0c5342b9e2c48f0fcfd145b5e2..23670e1ff9bac8689e40ef8bd75e38d15efcf530 100644 (file)
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1579,6 +1579,7 @@ typedef struct WorkTableScanState
 typedef struct ForeignScanState
 {
    ScanState   ss;             /* its first field is NodeTag */
+   List       *fdw_recheck_quals;  /* original quals not in ss.ps.qual */
    /* use struct pointer to avoid including fdwapi.h here */
    struct FdwRoutine *fdwroutine;
    void       *fdw_state;      /* foreign-data wrapper can keep state here */

diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 1f9213c09b0862959a95b1a1e839710d80f02ec6..92fd8e4fe4e11e2a8d2d283ce1d31f55bb9998de 100644 (file)
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -512,6 +512,11 @@ typedef struct WorkTableScan
  * fdw_scan_tlist is never actually executed; it just holds expression trees
  * describing what is in the scan tuple's columns.
  *
+ * fdw_recheck_quals should contain any quals which the core system passed to
+ * the FDW but which were not added to scan.plan.quals; that is, it should
+ * contain the quals being checked remotely.  This is needed for correct
+ * behavior during EvalPlanQual rechecks.
+ *
  * When the plan node represents a foreign join, scan.scanrelid is zero and
  * fs_relids must be consulted to identify the join relation.  (fs_relids
  * is valid for simple scans as well, but will always match scan.scanrelid.)
@@ -524,6 +529,7 @@ typedef struct ForeignScan
    List       *fdw_exprs;      /* expressions that FDW may evaluate */
    List       *fdw_private;    /* private data for FDW */
    List       *fdw_scan_tlist; /* optional tlist describing scan tuple */
+   List       *fdw_recheck_quals;  /* original quals not in scan.plan.quals */
    Bitmapset  *fs_relids;      /* RTIs generated by this scan */
    bool        fsSystemCol;    /* true if any "system column" is needed */
 } ForeignScan;

diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h
index 52b077a1b471516da3d4d4ada70d65afacb644f3..1fb850489fba014efdd772ad57847597a110c9bc 100644 (file)
--- a/src/include/optimizer/planmain.h
+++ b/src/include/optimizer/planmain.h
@@ -45,7 +45,7 @@ extern SubqueryScan *make_subqueryscan(List *qptlist, List *qpqual,
                  Index scanrelid, Plan *subplan);
 extern ForeignScan *make_foreignscan(List *qptlist, List *qpqual,
                 Index scanrelid, List *fdw_exprs, List *fdw_private,
-                List *fdw_scan_tlist);
+                List *fdw_scan_tlist, List *fdw_recheck_quals);
 extern Append *make_append(List *appendplans, List *tlist);
 extern RecursiveUnion *make_recursive_union(List *tlist,
                     Plan *lefttree, Plan *righttree, int wtParam,