Add regression test for 2PC visibility check on standby

This adds some test coverage for a defect fixed in 2e57790, where
the only reliable test back then was to use a hardcoded sleep().

This test relies on an injection point that is persisted across a node
restart, so as it is possible to cause the checkpointer to wait when
configuring the shared memory state of shared_preload_libraries at a
very early startup stage.

Reverting 2e57790 causes the test to fail, and it passes on HEAD.

Author: michaelpq

src/backend/replication/syncrep.c

@@ -84,6 +84,7 @@
 #include "storage/proc.h"
 #include "tcop/tcopprot.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/ps_status.h"
 
 /* User-settable parameters for sync rep */
@@ -968,6 +969,8 @@ SyncRepUpdateSyncStandbysDefined(void)
 	if (sync_standbys_defined !=
 		((WalSndCtl->sync_standbys_status & SYNC_STANDBY_DEFINED) != 0))
 	{
+		INJECTION_POINT("checkpointer-syncrep-update", NULL);
+
 		LWLockAcquire(SyncRepLock, LW_EXCLUSIVE);
 
 		/*

src/test/recovery/t/009_twophase.pl

@@ -51,6 +51,27 @@ sub configure_and_reload
 ));
 $node_paris->start;
 
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+my $injection_points_supported =
+  $node_london->check_extension('injection_points');
+if ($injection_points_supported != 0)
+{
+	$node_london->safe_psql('postgres', 'CREATE EXTENSION injection_points;');
+
+	# Set shared_preload_libraries, to allow the injection points to persist
+	# across restarts.
+	$node_london->append_conf(
+		'postgresql.conf', qq(
+		shared_preload_libraries = 'injection_points'
+	));
+	$node_paris->append_conf(
+		'postgresql.conf', qq(
+		shared_preload_libraries = 'injection_points'
+	));
+}
+
 # Switch to synchronous replication in both directions
 configure_and_reload($node_london, "synchronous_standby_names = 'paris'");
 configure_and_reload($node_paris, "synchronous_standby_names = 'london'");
@@ -327,6 +348,23 @@ sub configure_and_reload
 	INSERT INTO t_009_tbl_standby_mvcc VALUES (2, 'issued to ${cur_primary_name}');
 	PREPARE TRANSACTION 'xact_009_standby_mvcc';
 	");
+
+# Attach an injection point to wait in the checkpointer when configuring
+# the shared memory state data related to synchronous_standby_names, then
+# persist the attached point to disk so as the follow-up restart will be able
+# to wait at the early stages of the checkpointer startup sequence.
+#
+# Note that as the checkpointer has already applied the
+# synchronous_standby_names configuration, this has no effect until the
+# next startup of the primary.
+if ($injection_points_supported != 0)
+{
+	$cur_primary->psql('postgres',
+		"SELECT injection_points_attach('checkpointer-syncrep-update', 'wait')"
+	);
+	$cur_primary->psql('postgres', "SELECT injection_points_flush()");
+}
+
 $cur_primary->stop;
 $cur_standby->restart;
 
@@ -341,6 +379,16 @@ sub configure_and_reload
 
 # Commit the transaction in primary
 $cur_primary->start;
+
+# Make sure that the checkpointer is waiting before setting up the data of
+# synchronous_standby_names in shared memory.  We want the checkpointer to be
+# stuck and make sure that the next COMMIT PREPARED is detected correctly on
+# the standby when remote_apply is set on the primary.
+if ($injection_points_supported != 0)
+{
+	$cur_primary->wait_for_event('checkpointer',
+		'checkpointer-syncrep-update');
+}
 $cur_primary->psql(
 	'postgres', "
 SET synchronous_commit='remote_apply'; -- To ensure the standby is caught up
@@ -361,6 +409,18 @@ sub configure_and_reload
 	"Committed prepared transaction is visible to new snapshot in standby");
 $standby_session->quit;
 
+# Remove the injection point, the checkpointer now applies the configuration
+# related to synchronous_standby_names in shared memory.
+if ($injection_points_supported != 0)
+{
+	$cur_primary->psql('postgres',
+		"SELECT injection_points_wakeup('checkpointer-syncrep-update')");
+	$cur_primary->psql('postgres',
+		"SELECT injection_points_detach('checkpointer-syncrep-update')");
+}
+
+$cur_standby->restart;
+
 ###############################################################################
 # Check for a lock conflict between prepared transaction with DDL inside and
 # replay of XLOG_STANDBY_LOCK wal record.