snapshot scalability: cache snapshots using a xact completion counter.

Previous commits made it faster/more scalable to compute snapshots. But not
building a snapshot is still faster. Now that GetSnapshotData() does not
maintain RecentGlobal* anymore, that is actually not too hard:

This commit introduces xactCompletionCount, which tracks the number of
top-level transactions with xids (i.e. which may have modified the database)
that completed in some form since the start of the server.

We can avoid rebuilding the snapshot's contents whenever the current
xactCompletionCount is the same as it was when the snapshot was
originally built.  Currently this check happens while holding
ProcArrayLock. While it's likely possible to perform the check without
acquiring ProcArrayLock, it seems better to do that separately /
later, some careful analysis is required. Even with the lock this is a
significant win on its own.

On a smaller two socket machine this gains another ~1.03x, on a larger
machine the effect is roughly double (earlier patch version tested
though).  If we were able to safely avoid the lock there'd be another
significant gain on top of that.

Author: Andres Freund <[email protected]>
Reviewed-By: Robert Haas <[email protected]>
Reviewed-By: Thomas Munro <[email protected]>
Reviewed-By: David Rowley <[email protected]>
Discussion: https://postgr.es/m/20200301083601[email protected]

diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index e9701ea7221549f02fb683766cbf024577066ca7..9d5d68f3fa7855daa4782cb162c7620ddb85bfc0 100644 (file)
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -524,6 +524,7 @@ SnapBuildBuildSnapshot(SnapBuild *builder)
    snapshot->curcid = FirstCommandId;
    snapshot->active_count = 0;
    snapshot->regd_count = 0;
+   snapshot->snapXactCompletionCount = 0;
 
    return snapshot;
 }

diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 96e4a87857602f03f9c856edefaa16af5e22e636..e687cde6f176fcf955df020ca29f3e1489dad0a0 100644 (file)
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -407,6 +407,7 @@ CreateSharedProcArray(void)
        procArray->lastOverflowedXid = InvalidTransactionId;
        procArray->replication_slot_xmin = InvalidTransactionId;
        procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+       ShmemVariableCache->xactCompletionCount = 1;
    }
 
    allProcs = ProcGlobal->allProcs;
@@ -534,6 +535,9 @@ ProcArrayRemove(PGPROC *proc, TransactionId latestXid)
        /* Advance global latestCompletedXid while holding the lock */
        MaintainLatestCompletedXid(latestXid);
 
+       /* Same with xactCompletionCount  */
+       ShmemVariableCache->xactCompletionCount++;
+
        ProcGlobal->xids[proc->pgxactoff] = 0;
        ProcGlobal->subxidStates[proc->pgxactoff].overflowed = false;
        ProcGlobal->subxidStates[proc->pgxactoff].count = 0;
@@ -667,6 +671,7 @@ ProcArrayEndTransactionInternal(PGPROC *proc, TransactionId latestXid)
 {
    size_t      pgxactoff = proc->pgxactoff;
 
+   Assert(LWLockHeldByMe(ProcArrayLock));
    Assert(TransactionIdIsValid(ProcGlobal->xids[pgxactoff]));
    Assert(ProcGlobal->xids[pgxactoff] == proc->xid);
 
@@ -698,6 +703,9 @@ ProcArrayEndTransactionInternal(PGPROC *proc, TransactionId latestXid)
 
    /* Also advance global latestCompletedXid while holding the lock */
    MaintainLatestCompletedXid(latestXid);
+
+   /* Same with xactCompletionCount  */
+   ShmemVariableCache->xactCompletionCount++;
 }
 
 /*
@@ -1916,6 +1924,93 @@ GetMaxSnapshotSubxidCount(void)
    return TOTAL_MAX_CACHED_SUBXIDS;
 }
 
+/*
+ * Initialize old_snapshot_threshold specific parts of a newly build snapshot.
+ */
+static void
+GetSnapshotDataInitOldSnapshot(Snapshot snapshot)
+{
+   if (!OldSnapshotThresholdActive())
+   {
+       /*
+        * If not using "snapshot too old" feature, fill related fields with
+        * dummy values that don't require any locking.
+        */
+       snapshot->lsn = InvalidXLogRecPtr;
+       snapshot->whenTaken = 0;
+   }
+   else
+   {
+       /*
+        * Capture the current time and WAL stream location in case this
+        * snapshot becomes old enough to need to fall back on the special
+        * "old snapshot" logic.
+        */
+       snapshot->lsn = GetXLogInsertRecPtr();
+       snapshot->whenTaken = GetSnapshotCurrentTimestamp();
+       MaintainOldSnapshotTimeMapping(snapshot->whenTaken, snapshot->xmin);
+   }
+}
+
+/*
+ * Helper function for GetSnapshotData() that checks if the bulk of the
+ * visibility information in the snapshot is still valid. If so, it updates
+ * the fields that need to change and returns true. Otherwise it returns
+ * false.
+ *
+ * This very likely can be evolved to not need ProcArrayLock held (at very
+ * least in the case we already hold a snapshot), but that's for another day.
+ */
+static bool
+GetSnapshotDataReuse(Snapshot snapshot)
+{
+   uint64 curXactCompletionCount;
+
+   Assert(LWLockHeldByMe(ProcArrayLock));
+
+   if (unlikely(snapshot->snapXactCompletionCount == 0))
+       return false;
+
+   curXactCompletionCount = ShmemVariableCache->xactCompletionCount;
+   if (curXactCompletionCount != snapshot->snapXactCompletionCount)
+       return false;
+
+   /*
+    * If the current xactCompletionCount is still the same as it was at the
+    * time the snapshot was built, we can be sure that rebuilding the
+    * contents of the snapshot the hard way would result in the same snapshot
+    * contents:
+    *
+    * As explained in transam/README, the set of xids considered running by
+    * GetSnapshotData() cannot change while ProcArrayLock is held. Snapshot
+    * contents only depend on transactions with xids and xactCompletionCount
+    * is incremented whenever a transaction with an xid finishes (while
+    * holding ProcArrayLock) exclusively). Thus the xactCompletionCount check
+    * ensures we would detect if the snapshot would have changed.
+    *
+    * As the snapshot contents are the same as it was before, it is is safe
+    * to re-enter the snapshot's xmin into the PGPROC array. None of the rows
+    * visible under the snapshot could already have been removed (that'd
+    * require the set of running transactions to change) and it fulfills the
+    * requirement that concurrent GetSnapshotData() calls yield the same
+    * xmin.
+    */
+   if (!TransactionIdIsValid(MyProc->xmin))
+       MyProc->xmin = TransactionXmin = snapshot->xmin;
+
+   RecentXmin = snapshot->xmin;
+   Assert(TransactionIdPrecedesOrEquals(TransactionXmin, RecentXmin));
+
+   snapshot->curcid = GetCurrentCommandId(false);
+   snapshot->active_count = 0;
+   snapshot->regd_count = 0;
+   snapshot->copied = false;
+
+   GetSnapshotDataInitOldSnapshot(snapshot);
+
+   return true;
+}
+
 /*
  * GetSnapshotData -- returns information about running transactions.
  *
@@ -1963,6 +2058,7 @@ GetSnapshotData(Snapshot snapshot)
    TransactionId oldestxid;
    int         mypgxactoff;
    TransactionId myxid;
+   uint64      curXactCompletionCount;
 
    TransactionId replication_slot_xmin = InvalidTransactionId;
    TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
@@ -2007,12 +2103,19 @@ GetSnapshotData(Snapshot snapshot)
     */
    LWLockAcquire(ProcArrayLock, LW_SHARED);
 
+   if (GetSnapshotDataReuse(snapshot))
+   {
+       LWLockRelease(ProcArrayLock);
+       return snapshot;
+   }
+
    latest_completed = ShmemVariableCache->latestCompletedXid;
    mypgxactoff = MyProc->pgxactoff;
    myxid = other_xids[mypgxactoff];
    Assert(myxid == MyProc->xid);
 
    oldestxid = ShmemVariableCache->oldestXid;
+   curXactCompletionCount = ShmemVariableCache->xactCompletionCount;
 
    /* xmax is always latestCompletedXid + 1 */
    xmax = XidFromFullTransactionId(latest_completed);
@@ -2266,6 +2369,7 @@ GetSnapshotData(Snapshot snapshot)
    snapshot->xcnt = count;
    snapshot->subxcnt = subcount;
    snapshot->suboverflowed = suboverflowed;
+   snapshot->snapXactCompletionCount = curXactCompletionCount;
 
    snapshot->curcid = GetCurrentCommandId(false);
 
@@ -2277,26 +2381,7 @@ GetSnapshotData(Snapshot snapshot)
    snapshot->regd_count = 0;
    snapshot->copied = false;
 
-   if (old_snapshot_threshold < 0)
-   {
-       /*
-        * If not using "snapshot too old" feature, fill related fields with
-        * dummy values that don't require any locking.
-        */
-       snapshot->lsn = InvalidXLogRecPtr;
-       snapshot->whenTaken = 0;
-   }
-   else
-   {
-       /*
-        * Capture the current time and WAL stream location in case this
-        * snapshot becomes old enough to need to fall back on the special
-        * "old snapshot" logic.
-        */
-       snapshot->lsn = GetXLogInsertRecPtr();
-       snapshot->whenTaken = GetSnapshotCurrentTimestamp();
-       MaintainOldSnapshotTimeMapping(snapshot->whenTaken, xmin);
-   }
+   GetSnapshotDataInitOldSnapshot(snapshot);
 
    return snapshot;
 }

diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c
index c208538e2e5ca32d6e9ce1d8bb61fe56d54f704e..22cf3ebaf4728d0295fbed11dec3fe00e7e81d7a 100644 (file)
--- a/src/backend/utils/time/snapmgr.c
+++ b/src/backend/utils/time/snapmgr.c
@@ -597,6 +597,8 @@ SetTransactionSnapshot(Snapshot sourcesnap, VirtualTransactionId *sourcevxid,
    CurrentSnapshot->takenDuringRecovery = sourcesnap->takenDuringRecovery;
    /* NB: curcid should NOT be copied, it's a local matter */
 
+   CurrentSnapshot->snapXactCompletionCount = 0;
+
    /*
     * Now we have to fix what GetSnapshotData did with MyProc->xmin and
     * TransactionXmin.  There is a race condition: to make sure we are not
@@ -672,6 +674,7 @@ CopySnapshot(Snapshot snapshot)
    newsnap->regd_count = 0;
    newsnap->active_count = 0;
    newsnap->copied = true;
+   newsnap->snapXactCompletionCount = 0;
 
    /* setup XID array */
    if (snapshot->xcnt > 0)
@@ -2209,6 +2212,7 @@ RestoreSnapshot(char *start_address)
    snapshot->curcid = serialized_snapshot.curcid;
    snapshot->whenTaken = serialized_snapshot.whenTaken;
    snapshot->lsn = serialized_snapshot.lsn;
+   snapshot->snapXactCompletionCount = 0;
 
    /* Copy XIDs, if present. */
    if (serialized_snapshot.xcnt > 0)

diff --git a/src/include/access/transam.h b/src/include/access/transam.h
index b32044153b09d9fac6264c4812427294d624360f..2f1f144db4d062a374104bc62d382d0d5c7c4de7 100644 (file)
--- a/src/include/access/transam.h
+++ b/src/include/access/transam.h
@@ -231,6 +231,15 @@ typedef struct VariableCacheData
    FullTransactionId latestCompletedXid;   /* newest full XID that has
                                             * committed or aborted */
 
+   /*
+    * Number of top-level transactions with xids (i.e. which may have
+    * modified the database) that completed in some form since the start of
+    * the server. This currently is solely used to check whether
+    * GetSnapshotData() needs to recompute the contents of the snapshot, or
+    * not. There are likely other users of this.  Always above 1.
+    */
+   uint64 xactCompletionCount;
+
    /*
     * These fields are protected by XactTruncationLock
     */

diff --git a/src/include/utils/snapshot.h b/src/include/utils/snapshot.h
index 35b1f05bea6593c6c5df28b3fb3bbf51b132c6a0..dea072e5edf5e881e172cebb81a5c8c29398e65c 100644 (file)
--- a/src/include/utils/snapshot.h
+++ b/src/include/utils/snapshot.h
@@ -207,6 +207,13 @@ typedef struct SnapshotData
 
    TimestampTz whenTaken;      /* timestamp when snapshot was taken */
    XLogRecPtr  lsn;            /* position in the WAL stream when taken */
+
+   /*
+    * The transaction completion count at the time GetSnapshotData() built
+    * this snapshot. Allows to avoid re-computing static snapshots when no
+    * transactions completed since the last GetSnapshotData().
+    */
+   uint64      snapXactCompletionCount;
 } SnapshotData;
 
 #endif                         /* SNAPSHOT_H */