From: Alvaro Herrera Date: Tue, 2 Apr 2019 18:18:08 +0000 (-0300) Subject: Report progress of CREATE INDEX operations X-Git-Tag: REL_12_BETA1~331 X-Git-Url: http://git.postgresql.org/gitweb/?a=commitdiff_plain;h=ab0dfc961b6a821f23d9c40c723d11380ce195a6;p=postgresql.git Report progress of CREATE INDEX operations This uses the progress reporting infrastructure added by c16dc1aca5e0, adding support for CREATE INDEX and CREATE INDEX CONCURRENTLY. There are two pieces to this: one is index-AM-agnostic, and the other is AM-specific. The latter is fairly elaborate for btrees, including reportage for parallel index builds and the separate phases that btree index creation uses; other index AMs, which are much simpler in their building procedures, have simplistic reporting only, but that seems sufficient, at least for non-concurrent builds. The index-AM-agnostic part is fairly complete, providing insight into the CONCURRENTLY wait phases as well as block-based progress during the index validation table scan. (The index validation index scan requires patching each AM, which has not been included here.) Reviewers: Rahila Syed, Pavan Deolasee, Tatsuro Yamada Discussion: https://postgr.es/m/20181220220022.mg63bhk26zdpvmcj@alvherre.pgsql --- diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c index 9d5b2e5be67..591e0a3e46a 100644 --- a/contrib/amcheck/verify_nbtree.c +++ b/contrib/amcheck/verify_nbtree.c @@ -566,7 +566,7 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace, RelationGetRelationName(state->rel), RelationGetRelationName(state->heaprel)); - table_index_build_scan(state->heaprel, state->rel, indexinfo, true, + table_index_build_scan(state->heaprel, state->rel, indexinfo, true, false, bt_tuple_present_callback, (void *) state, scan); ereport(DEBUG1, diff --git a/contrib/bloom/blinsert.c b/contrib/bloom/blinsert.c index 1b8df7e1e84..4b2186b8dda 100644 --- a/contrib/bloom/blinsert.c +++ b/contrib/bloom/blinsert.c @@ -142,7 +142,7 @@ blbuild(Relation heap, Relation index, IndexInfo *indexInfo) initCachedPage(&buildstate); /* Do the heap scan */ - reltuples = table_index_build_scan(heap, index, indexInfo, true, + reltuples = table_index_build_scan(heap, index, indexInfo, true, true, bloomBuildCallback, (void *) &buildstate, NULL); diff --git a/contrib/bloom/blutils.c b/contrib/bloom/blutils.c index d078dfbd469..ee3bd562748 100644 --- a/contrib/bloom/blutils.c +++ b/contrib/bloom/blutils.c @@ -132,6 +132,7 @@ blhandler(PG_FUNCTION_ARGS) amroutine->amcostestimate = blcostestimate; amroutine->amoptions = bloptions; amroutine->amproperty = NULL; + amroutine->ambuildphasename = NULL; amroutine->amvalidate = blvalidate; amroutine->ambeginscan = blbeginscan; amroutine->amrescan = blrescan; diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index b56d3b3daa1..ff8290da9ff 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -127,6 +127,7 @@ typedef struct IndexAmRoutine amcostestimate_function amcostestimate; amoptions_function amoptions; amproperty_function amproperty; /* can be NULL */ + ambuildphasename_function ambuildphasename; /* can be NULL */ amvalidate_function amvalidate; ambeginscan_function ambeginscan; amrescan_function amrescan; @@ -468,6 +469,18 @@ amproperty (Oid index_oid, int attno, +char * +ambuildphasename (int64 phasenum); + + Return the textual name of the given build phase number. + The phase numbers are those reported during an index build via the + pgstat_progress_update_param interface. + The phase names are then exposed in the + pg_stat_progress_create_index view. + + + + bool amvalidate (Oid opclassoid); diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f1df14bdea8..66792605082 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -336,6 +336,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_progress_create_indexpg_stat_progress_create_index + One row for each backend running CREATE INDEX, showing + current progress. + See . + + + pg_stat_progress_vacuumpg_stat_progress_vacuum One row for each backend (including autovacuum worker processes) running @@ -3403,10 +3411,224 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, PostgreSQL has the ability to report the progress of certain commands during command execution. Currently, the only commands - which support progress reporting are VACUUM and + which support progress reporting are CREATE INDEX, + VACUUM and CLUSTER. This may be expanded in the future. + + CREATE INDEX Progress Reporting + + + Whenever CREATE INDEX is running, the + pg_stat_progress_create_index view will contain + one row for each backend that is currently creating indexes. The tables + below describe the information that will be reported and provide information + about how to interpret it. + + + + <structname>pg_stat_progress_create_index</structname> View + + + + Column + Type + Description + + + + + + pid + integer + Process ID of backend. + + + datid + oid + OID of the database to which this backend is connected. + + + datname + name + Name of the database to which this backend is connected. + + + relid + oid + OID of the table on which the index is being created. + + + phase + text + + Current processing phase of index creation. See . + + + + lockers_total + bigint + + Total number of lockers to wait for, when applicable. + + + + lockers_done + bigint + + Number of lockers already waited for. + + + + current_locked_pid + bigint + + Process ID of the locker currently being waited for. + + + + blocks_total + bigint + + Total number of blocks to be processed in the current phase. + + + + blocks_done + bigint + + Number of blocks already processed in the current phase. + + + + tuples_total + bigint + + Total number of tuples to be processed in the current phase. + + + + tuples_done + bigint + + Number of tuples already processed in the current phase. + + + + partitions_total + bigint + + When creating an index on a partitioned table, this column is set to + the total number of partitions on which the index is to be created. + + + + partitions_done + bigint + + When creating an index on a partitioned table, this column is set to + the number of partitions on which the index has been completed. + + + + +
+ + + CREATE INDEX phases + + + + Phase + Description + + + + + initializing + + CREATE INDEX is preparing to create the index. This + phase is expected to be very brief. + + + + waiting for old snapshots + + CREATE INDEX CONCURRENTLY is waiting for transactions + that can potentially see the table to release their snapshots. + This phase is skipped when not in concurrent mode. + Columns lockers_total, lockers_done + and current_locker_pid contain the progress + information for this phase. + + + + building index + + The index is being built by the access method-specific code. In this phase, + access methods that support progress reporting fill in their own progress data, + and the subphase is indicated in this column. Typically, + blocks_total and blocks_done + will contain progress data, as well as potentially + tuples_total and tuples_done. + + + + waiting for writer snapshots + + CREATE INDEX CONCURRENTLY is waiting for transactions + that can potentially write into the table to release their snapshots. + This phase is skipped when not in concurrent mode. + Columns lockers_total, lockers_done + and current_locker_pid contain the progress + information for this phase. + + + + index validation: scanning index + + CREATE INDEX CONCURRENTLY is scanning the index searching + for tuples that need to be validated. + This phase is skipped when not in concurrent mode. + Columns blocks_total (set to the total size of the index) + and blocks_done contain the progress information for this phase. + + + + index validation: sorting tuples + + CREATE INDEX CONCURRENTLY is sorting the output of the + index scanning phase. + + + + index validation: scanning table + + CREATE INDEX CONCURRENTLY is scanning the table + to validate the index tuples collected in the previous two phases. + This phase is skipped when not in concurrent mode. + Columns blocks_total (set to the total size of the table) + and blocks_done contain the progress information for this phase. + + + + waiting for reader snapshots + + CREATE INDEX CONCURRENTLY is waiting for transactions + that can potentially see the table to release their snapshots. This + phase is skipped when not in concurrent mode. + Columns lockers_total, lockers_done + and current_locker_pid contain the progress + information for this phase. + + + + +
+ + + VACUUM Progress Reporting diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index 6e96d24ca22..5c2b0c76358 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -112,6 +112,7 @@ brinhandler(PG_FUNCTION_ARGS) amroutine->amcostestimate = brincostestimate; amroutine->amoptions = brinoptions; amroutine->amproperty = NULL; + amroutine->ambuildphasename = NULL; amroutine->amvalidate = brinvalidate; amroutine->ambeginscan = brinbeginscan; amroutine->amrescan = brinrescan; @@ -719,7 +720,7 @@ brinbuild(Relation heap, Relation index, IndexInfo *indexInfo) * Now scan the relation. No syncscan allowed here because we want the * heap blocks in physical order. */ - reltuples = table_index_build_scan(heap, index, indexInfo, false, + reltuples = table_index_build_scan(heap, index, indexInfo, false, true, brinbuildCallback, (void *) state, NULL); /* process the final batch */ @@ -1236,7 +1237,7 @@ summarize_range(IndexInfo *indexInfo, BrinBuildState *state, Relation heapRel, * cases. */ state->bs_currRangeStart = heapBlk; - table_index_build_range_scan(heapRel, state->bs_irel, indexInfo, false, true, + table_index_build_range_scan(heapRel, state->bs_irel, indexInfo, false, true, false, heapBlk, scanNumBlks, brinbuildCallback, (void *) state, NULL); diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c index b02f69b0dcb..edc353a7fe0 100644 --- a/src/backend/access/gin/gininsert.c +++ b/src/backend/access/gin/gininsert.c @@ -395,7 +395,7 @@ ginbuild(Relation heap, Relation index, IndexInfo *indexInfo) * Do the heap scan. We disallow sync scan here because dataPlaceToPage * prefers to receive tuples in TID order. */ - reltuples = table_index_build_scan(heap, index, indexInfo, false, + reltuples = table_index_build_scan(heap, index, indexInfo, false, true, ginBuildCallback, (void *) &buildstate, NULL); diff --git a/src/backend/access/gin/ginutil.c b/src/backend/access/gin/ginutil.c index afc20232ace..d2360eeafb0 100644 --- a/src/backend/access/gin/ginutil.c +++ b/src/backend/access/gin/ginutil.c @@ -64,6 +64,7 @@ ginhandler(PG_FUNCTION_ARGS) amroutine->amcostestimate = gincostestimate; amroutine->amoptions = ginoptions; amroutine->amproperty = NULL; + amroutine->ambuildphasename = NULL; amroutine->amvalidate = ginvalidate; amroutine->ambeginscan = ginbeginscan; amroutine->amrescan = ginrescan; diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index 2fddb23496d..f44c922b5d6 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -86,6 +86,7 @@ gisthandler(PG_FUNCTION_ARGS) amroutine->amcostestimate = gistcostestimate; amroutine->amoptions = gistoptions; amroutine->amproperty = gistproperty; + amroutine->ambuildphasename = NULL; amroutine->amvalidate = gistvalidate; amroutine->ambeginscan = gistbeginscan; amroutine->amrescan = gistrescan; diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c index 3652fde5bb1..6024671989e 100644 --- a/src/backend/access/gist/gistbuild.c +++ b/src/backend/access/gist/gistbuild.c @@ -205,7 +205,7 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) /* * Do the heap scan. */ - reltuples = table_index_build_scan(heap, index, indexInfo, true, + reltuples = table_index_build_scan(heap, index, indexInfo, true, true, gistBuildCallback, (void *) &buildstate, NULL); diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index 5cc12a17130..048e40e46fa 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -23,9 +23,11 @@ #include "access/relscan.h" #include "access/tableam.h" #include "catalog/index.h" +#include "commands/progress.h" #include "commands/vacuum.h" #include "miscadmin.h" #include "optimizer/plancat.h" +#include "pgstat.h" #include "utils/builtins.h" #include "utils/index_selfuncs.h" #include "utils/rel.h" @@ -83,6 +85,7 @@ hashhandler(PG_FUNCTION_ARGS) amroutine->amcostestimate = hashcostestimate; amroutine->amoptions = hashoptions; amroutine->amproperty = NULL; + amroutine->ambuildphasename = NULL; amroutine->amvalidate = hashvalidate; amroutine->ambeginscan = hashbeginscan; amroutine->amrescan = hashrescan; @@ -160,9 +163,11 @@ hashbuild(Relation heap, Relation index, IndexInfo *indexInfo) buildstate.heapRel = heap; /* do the heap scan */ - reltuples = table_index_build_scan(heap, index, indexInfo, true, + reltuples = table_index_build_scan(heap, index, indexInfo, true, true, hashbuildCallback, (void *) &buildstate, NULL); + pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_TOTAL, + buildstate.indtuples); if (buildstate.spool) { diff --git a/src/backend/access/hash/hashsort.c b/src/backend/access/hash/hashsort.c index 8c55436b193..00a57470a77 100644 --- a/src/backend/access/hash/hashsort.c +++ b/src/backend/access/hash/hashsort.c @@ -26,7 +26,9 @@ #include "postgres.h" #include "access/hash.h" +#include "commands/progress.h" #include "miscadmin.h" +#include "pgstat.h" #include "utils/tuplesort.h" @@ -116,6 +118,7 @@ void _h_indexbuild(HSpool *hspool, Relation heapRel) { IndexTuple itup; + long tups_done = 0; #ifdef USE_ASSERT_CHECKING uint32 hashkey = 0; #endif @@ -141,5 +144,8 @@ _h_indexbuild(HSpool *hspool, Relation heapRel) #endif _hash_doinsert(hspool->index, itup, heapRel); + + pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE, + ++tups_done); } } diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 5c96fc91b79..6693d7eb2dd 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -57,6 +57,8 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, HeapTuple tuple, OffsetNumber tupoffset); +static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); + static const TableAmRoutine heapam_methods; @@ -1120,6 +1122,7 @@ heapam_index_build_range_scan(Relation heapRelation, IndexInfo *indexInfo, bool allow_sync, bool anyvisible, + bool progress, BlockNumber start_blockno, BlockNumber numblocks, IndexBuildCallback callback, @@ -1140,6 +1143,7 @@ heapam_index_build_range_scan(Relation heapRelation, Snapshot snapshot; bool need_unregister_snapshot = false; TransactionId OldestXmin; + BlockNumber previous_blkno = InvalidBlockNumber; BlockNumber root_blkno = InvalidBlockNumber; OffsetNumber root_offsets[MaxHeapTuplesPerPage]; @@ -1227,6 +1231,25 @@ heapam_index_build_range_scan(Relation heapRelation, hscan = (HeapScanDesc) scan; + /* Publish number of blocks to scan */ + if (progress) + { + BlockNumber nblocks; + + if (hscan->rs_base.rs_parallel != NULL) + { + ParallelBlockTableScanDesc pbscan; + + pbscan = (ParallelBlockTableScanDesc) hscan->rs_base.rs_parallel; + nblocks = pbscan->phs_nblocks; + } + else + nblocks = hscan->rs_nblocks; + + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + nblocks); + } + /* * Must call GetOldestXmin() with SnapshotAny. Should never call * GetOldestXmin() with MVCC snapshot. (It's especially worth checking @@ -1259,6 +1282,19 @@ heapam_index_build_range_scan(Relation heapRelation, CHECK_FOR_INTERRUPTS(); + /* Report scan progress, if asked to. */ + if (progress) + { + BlockNumber blocks_done = heapam_scan_get_blocks_done(hscan); + + if (blocks_done != previous_blkno) + { + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blocks_done); + previous_blkno = blocks_done; + } + } + /* * When dealing with a HOT-chain of updated tuples, we want to index * the values of the live tuple (if any), but index it under the TID @@ -1600,6 +1636,25 @@ heapam_index_build_range_scan(Relation heapRelation, } } + /* Report scan progress one last time. */ + if (progress) + { + BlockNumber blks_done; + + if (hscan->rs_base.rs_parallel != NULL) + { + ParallelBlockTableScanDesc pbscan; + + pbscan = (ParallelBlockTableScanDesc) hscan->rs_base.rs_parallel; + blks_done = pbscan->phs_nblocks; + } + else + blks_done = hscan->rs_nblocks; + + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blks_done); + } + table_endscan(scan); /* we can now forget our snapshot, if set and registered by us */ @@ -1636,6 +1691,7 @@ heapam_index_validate_scan(Relation heapRelation, BlockNumber root_blkno = InvalidBlockNumber; OffsetNumber root_offsets[MaxHeapTuplesPerPage]; bool in_index[MaxHeapTuplesPerPage]; + BlockNumber previous_blkno = InvalidBlockNumber; /* state variables for the merge */ ItemPointer indexcursor = NULL; @@ -1676,6 +1732,9 @@ heapam_index_validate_scan(Relation heapRelation, false); /* syncscan not OK */ hscan = (HeapScanDesc) scan; + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + hscan->rs_nblocks); + /* * Scan all tuples matching the snapshot. */ @@ -1689,6 +1748,14 @@ heapam_index_validate_scan(Relation heapRelation, state->htups += 1; + if ((previous_blkno == InvalidBlockNumber) || + (hscan->rs_cblock != previous_blkno)) + { + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + hscan->rs_cblock); + previous_blkno = hscan->rs_cblock; + } + /* * As commented in table_index_build_scan, we should index heap-only * tuples under the TIDs of their root tuples; so when we advance onto @@ -1849,6 +1916,46 @@ heapam_index_validate_scan(Relation heapRelation, indexInfo->ii_PredicateState = NULL; } +/* + * Return the number of blocks that have been read by this scan since + * starting. This is meant for progress reporting rather than be fully + * accurate: in a parallel scan, workers can be concurrently reading blocks + * further ahead than what we report. + */ +static BlockNumber +heapam_scan_get_blocks_done(HeapScanDesc hscan) +{ + ParallelBlockTableScanDesc bpscan = NULL; + BlockNumber startblock; + BlockNumber blocks_done; + + if (hscan->rs_base.rs_parallel != NULL) + { + bpscan = (ParallelBlockTableScanDesc) hscan->rs_base.rs_parallel; + startblock = bpscan->phs_startblock; + } + else + startblock = hscan->rs_startblock; + + /* + * Might have wrapped around the end of the relation, if startblock was + * not zero. + */ + if (hscan->rs_cblock > startblock) + blocks_done = hscan->rs_cblock - startblock; + else + { + BlockNumber nblocks; + + nblocks = bpscan != NULL ? bpscan->phs_nblocks : hscan->rs_nblocks; + blocks_done = nblocks - startblock + + hscan->rs_cblock; + } + + return blocks_done; +} + + /* ------------------------------------------------------------------------ * Planner related callbacks for the heap AM diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index ac6f1eb3423..7370379c6a1 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -22,6 +22,7 @@ #include "access/nbtxlog.h" #include "access/relscan.h" #include "access/xlog.h" +#include "commands/progress.h" #include "commands/vacuum.h" #include "miscadmin.h" #include "nodes/execnodes.h" @@ -133,6 +134,7 @@ bthandler(PG_FUNCTION_ARGS) amroutine->amcostestimate = btcostestimate; amroutine->amoptions = btoptions; amroutine->amproperty = btproperty; + amroutine->ambuildphasename = btbuildphasename; amroutine->amvalidate = btvalidate; amroutine->ambeginscan = btbeginscan; amroutine->amrescan = btrescan; @@ -1021,6 +1023,10 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, if (needLock) UnlockRelationForExtension(rel, ExclusiveLock); + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL, + num_pages); + /* Quit if we've scanned the whole relation */ if (blkno >= num_pages) break; @@ -1028,6 +1034,9 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, for (; blkno < num_pages; blkno++) { btvacuumpage(&vstate, blkno, blkno); + if (info->report_progress) + pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, + blkno); } } diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c index 14d95457682..9ac4c1e1c08 100644 --- a/src/backend/access/nbtree/nbtsort.c +++ b/src/backend/access/nbtree/nbtsort.c @@ -66,6 +66,7 @@ #include "access/xlog.h" #include "access/xloginsert.h" #include "catalog/index.h" +#include "commands/progress.h" #include "miscadmin.h" #include "pgstat.h" #include "storage/smgr.h" @@ -298,7 +299,8 @@ static double _bt_parallel_heapscan(BTBuildState *buildstate, static void _bt_leader_participate_as_worker(BTBuildState *buildstate); static void _bt_parallel_scan_and_sort(BTSpool *btspool, BTSpool *btspool2, BTShared *btshared, Sharedsort *sharedsort, - Sharedsort *sharedsort2, int sortmem); + Sharedsort *sharedsort2, int sortmem, + bool progress); /* @@ -394,6 +396,10 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate, /* Save as primary spool */ buildstate->spool = btspool; + /* Report table scan phase started */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, + PROGRESS_BTREE_PHASE_INDEXBUILD_TABLESCAN); + /* Attempt to launch parallel worker scan when required */ if (indexInfo->ii_ParallelWorkers > 0) _bt_begin_parallel(buildstate, indexInfo->ii_Concurrent, @@ -480,13 +486,31 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate, /* Fill spool using either serial or parallel heap scan */ if (!buildstate->btleader) - reltuples = table_index_build_scan(heap, index, indexInfo, true, + reltuples = table_index_build_scan(heap, index, indexInfo, true, true, _bt_build_callback, (void *) buildstate, NULL); else reltuples = _bt_parallel_heapscan(buildstate, &indexInfo->ii_BrokenHotChain); + /* + * Set the progress target for the next phase. Reset the block number + * values set by table_index_build_scan + */ + { + const int index[] = { + PROGRESS_CREATEIDX_TUPLES_TOTAL, + PROGRESS_SCAN_BLOCKS_TOTAL, + PROGRESS_SCAN_BLOCKS_DONE + }; + const int64 val[] = { + buildstate->indtuples, + 0, 0 + }; + + pgstat_progress_update_multi_param(3, index, val); + } + /* okay, all heap tuples are spooled */ if (buildstate->spool2 && !buildstate->havedead) { @@ -535,9 +559,15 @@ _bt_leafbuild(BTSpool *btspool, BTSpool *btspool2) } #endif /* BTREE_BUILD_STATS */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, + PROGRESS_BTREE_PHASE_PERFORMSORT_1); tuplesort_performsort(btspool->sortstate); if (btspool2) + { + pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, + PROGRESS_BTREE_PHASE_PERFORMSORT_2); tuplesort_performsort(btspool2->sortstate); + } wstate.heap = btspool->heap; wstate.index = btspool->index; @@ -554,6 +584,8 @@ _bt_leafbuild(BTSpool *btspool, BTSpool *btspool2) wstate.btws_pages_written = 0; wstate.btws_zeropage = NULL; /* until needed */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, + PROGRESS_BTREE_PHASE_LEAF_LOAD); _bt_load(&wstate, btspool, btspool2); } @@ -1098,6 +1130,7 @@ _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2) int i, keysz = IndexRelationGetNumberOfKeyAttributes(wstate->index); SortSupport sortKeys; + long tuples_done = 0; if (merge) { @@ -1202,6 +1235,10 @@ _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2) _bt_buildadd(wstate, state, itup2); itup2 = tuplesort_getindextuple(btspool2->sortstate, true); } + + /* Report progress */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE, + ++tuples_done); } pfree(sortKeys); } @@ -1216,6 +1253,10 @@ _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2) state = _bt_pagestate(wstate, 0); _bt_buildadd(wstate, state, itup); + + /* Report progress */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE, + ++tuples_done); } } @@ -1528,7 +1569,7 @@ _bt_leader_participate_as_worker(BTBuildState *buildstate) /* Perform work common to all participants */ _bt_parallel_scan_and_sort(leaderworker, leaderworker2, btleader->btshared, btleader->sharedsort, btleader->sharedsort2, - sortmem); + sortmem, true); #ifdef BTREE_BUILD_STATS if (log_btree_build_stats) @@ -1619,7 +1660,7 @@ _bt_parallel_build_main(dsm_segment *seg, shm_toc *toc) /* Perform sorting of spool, and possibly a spool2 */ sortmem = maintenance_work_mem / btshared->scantuplesortstates; _bt_parallel_scan_and_sort(btspool, btspool2, btshared, sharedsort, - sharedsort2, sortmem); + sharedsort2, sortmem, false); #ifdef BTREE_BUILD_STATS if (log_btree_build_stats) @@ -1648,7 +1689,7 @@ _bt_parallel_build_main(dsm_segment *seg, shm_toc *toc) static void _bt_parallel_scan_and_sort(BTSpool *btspool, BTSpool *btspool2, BTShared *btshared, Sharedsort *sharedsort, - Sharedsort *sharedsort2, int sortmem) + Sharedsort *sharedsort2, int sortmem, bool progress) { SortCoordinate coordinate; BTBuildState buildstate; @@ -1705,9 +1746,10 @@ _bt_parallel_scan_and_sort(BTSpool *btspool, BTSpool *btspool2, /* Join parallel scan */ indexInfo = BuildIndexInfo(btspool->index); indexInfo->ii_Concurrent = btshared->isconcurrent; - scan = table_beginscan_parallel(btspool->heap, ParallelTableScanFromBTShared(btshared)); + scan = table_beginscan_parallel(btspool->heap, + ParallelTableScanFromBTShared(btshared)); reltuples = table_index_build_scan(btspool->heap, btspool->index, indexInfo, - true, _bt_build_callback, + true, progress, _bt_build_callback, (void *) &buildstate, scan); /* diff --git a/src/backend/access/nbtree/nbtutils.c b/src/backend/access/nbtree/nbtutils.c index 140ac920265..7e409d616fe 100644 --- a/src/backend/access/nbtree/nbtutils.c +++ b/src/backend/access/nbtree/nbtutils.c @@ -20,6 +20,7 @@ #include "access/nbtree.h" #include "access/reloptions.h" #include "access/relscan.h" +#include "commands/progress.h" #include "miscadmin.h" #include "utils/array.h" #include "utils/datum.h" @@ -2051,6 +2052,29 @@ btproperty(Oid index_oid, int attno, } } +/* + * btbuildphasename() -- Return name of index build phase. + */ +char * +btbuildphasename(int64 phasenum) +{ + switch (phasenum) + { + case PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE: + return "initializing"; + case PROGRESS_BTREE_PHASE_INDEXBUILD_TABLESCAN: + return "scanning table"; + case PROGRESS_BTREE_PHASE_PERFORMSORT_1: + return "sorting live tuples"; + case PROGRESS_BTREE_PHASE_PERFORMSORT_2: + return "sorting dead tuples"; + case PROGRESS_BTREE_PHASE_LEAF_LOAD: + return "loading tuples in tree"; + default: + return NULL; + } +} + /* * _bt_truncate() -- create tuple without unneeded suffix attributes. * diff --git a/src/backend/access/spgist/spginsert.c b/src/backend/access/spgist/spginsert.c index 390ad9ac51f..b06feafdc24 100644 --- a/src/backend/access/spgist/spginsert.c +++ b/src/backend/access/spgist/spginsert.c @@ -143,7 +143,7 @@ spgbuild(Relation heap, Relation index, IndexInfo *indexInfo) "SP-GiST build temporary context", ALLOCSET_DEFAULT_SIZES); - reltuples = table_index_build_scan(heap, index, indexInfo, true, + reltuples = table_index_build_scan(heap, index, indexInfo, true, true, spgistBuildCallback, (void *) &buildstate, NULL); diff --git a/src/backend/access/spgist/spgutils.c b/src/backend/access/spgist/spgutils.c index 8e63c1fad25..45472db147b 100644 --- a/src/backend/access/spgist/spgutils.c +++ b/src/backend/access/spgist/spgutils.c @@ -67,6 +67,7 @@ spghandler(PG_FUNCTION_ARGS) amroutine->amcostestimate = spgcostestimate; amroutine->amoptions = spgoptions; amroutine->amproperty = spgproperty; + amroutine->ambuildphasename = NULL; amroutine->amvalidate = spgvalidate; amroutine->ambeginscan = spgbeginscan; amroutine->amrescan = spgrescan; diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 0d9d405c548..2ed7fdb021f 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -51,9 +51,9 @@ #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "catalog/storage.h" -#include "commands/tablecmds.h" #include "commands/event_trigger.h" #include "commands/progress.h" +#include "commands/tablecmds.h" #include "commands/trigger.h" #include "executor/executor.h" #include "miscadmin.h" @@ -2047,7 +2047,7 @@ index_drop(Oid indexId, bool concurrent, bool concurrent_lock_mode) * to acquire an exclusive lock on our table. The lock code will * detect deadlock and error out properly. */ - WaitForLockers(heaplocktag, AccessExclusiveLock); + WaitForLockers(heaplocktag, AccessExclusiveLock, true); /* Finish invalidation of index and mark it as dead */ index_concurrently_set_dead(heapId, indexId); @@ -2063,7 +2063,7 @@ index_drop(Oid indexId, bool concurrent, bool concurrent_lock_mode) * Wait till every transaction that saw the old index state has * finished. */ - WaitForLockers(heaplocktag, AccessExclusiveLock); + WaitForLockers(heaplocktag, AccessExclusiveLock, true); /* * Re-open relations to allow us to complete our actions. @@ -2712,6 +2712,25 @@ index_build(Relation heapRelation, save_sec_context | SECURITY_RESTRICTED_OPERATION); save_nestlevel = NewGUCNestLevel(); + /* Set up initial progress report status */ + { + const int index[] = { + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_SUBPHASE, + PROGRESS_CREATEIDX_TUPLES_DONE, + PROGRESS_CREATEIDX_TUPLES_TOTAL, + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + const int64 val[] = { + PROGRESS_CREATEIDX_PHASE_BUILD, + PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE, + 0, 0, 0, 0 + }; + + pgstat_progress_update_multi_param(6, index, val); + } + /* * Call the access method's build procedure */ @@ -3000,6 +3019,21 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot) int save_sec_context; int save_nestlevel; + { + const int index[] = { + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_TUPLES_DONE, + PROGRESS_CREATEIDX_TUPLES_TOTAL, + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + const int64 val[] = { + PROGRESS_CREATEIDX_PHASE_VALIDATE_IDXSCAN, + 0, 0, 0, 0 + }; + pgstat_progress_update_multi_param(5, index, val); + } + /* Open and lock the parent heap relation */ heapRelation = table_open(heapId, ShareUpdateExclusiveLock); /* And the target index relation */ @@ -3030,6 +3064,7 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot) */ ivinfo.index = indexRelation; ivinfo.analyze_only = false; + ivinfo.report_progress = true; ivinfo.estimated_count = true; ivinfo.message_level = DEBUG2; ivinfo.num_heap_tuples = heapRelation->rd_rel->reltuples; @@ -3047,15 +3082,31 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot) NULL, false); state.htups = state.itups = state.tups_inserted = 0; + /* ambulkdelete updates progress metrics */ (void) index_bulk_delete(&ivinfo, NULL, validate_index_callback, (void *) &state); /* Execute the sort */ + { + const int index[] = { + PROGRESS_CREATEIDX_PHASE, + PROGRESS_SCAN_BLOCKS_DONE, + PROGRESS_SCAN_BLOCKS_TOTAL + }; + const int64 val[] = { + PROGRESS_CREATEIDX_PHASE_VALIDATE_SORT, + 0, 0 + }; + + pgstat_progress_update_multi_param(3, index, val); + } tuplesort_performsort(state.tuplesort); /* * Now scan the heap and "merge" it with the index */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_VALIDATE_TABLESCAN); table_index_validate_scan(heapRelation, indexRelation, indexInfo, diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b89df70653e..3f2a7ef0158 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -934,6 +934,33 @@ CREATE VIEW pg_stat_progress_cluster AS FROM pg_stat_get_progress_info('CLUSTER') AS S LEFT JOIN pg_database D ON S.datid = D.oid; +CREATE VIEW pg_stat_progress_create_index AS + SELECT + S.pid AS pid, S.datid AS datid, D.datname AS datname, + S.relid AS relid, + CASE S.param10 WHEN 0 THEN 'initializing' + WHEN 1 THEN 'waiting for old snapshots' + WHEN 2 THEN 'building index' || + COALESCE((': ' || pg_indexam_progress_phasename(S.param9::oid, S.param11)), + '') + WHEN 3 THEN 'waiting for writer snapshots' + WHEN 4 THEN 'index validation: scanning index' + WHEN 5 THEN 'index validation: sorting tuples' + WHEN 6 THEN 'index validation: scanning table' + WHEN 7 THEN 'waiting for reader snapshots' + END as phase, + S.param4 AS lockers_total, + S.param5 AS lockers_done, + S.param6 AS current_locker_pid, + S.param16 AS blocks_total, + S.param17 AS blocks_done, + S.param12 AS tuples_total, + S.param13 AS tuples_done, + S.param14 AS partitions_total, + S.param15 AS partitions_done + FROM pg_stat_get_progress_info('CREATE INDEX') AS S + LEFT JOIN pg_database D ON S.datid = D.oid; + CREATE VIEW pg_user_mappings AS SELECT U.oid AS umid, diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 53971fc7258..348d5432977 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -36,6 +36,7 @@ #include "commands/dbcommands.h" #include "commands/defrem.h" #include "commands/event_trigger.h" +#include "commands/progress.h" #include "commands/tablecmds.h" #include "commands/tablespace.h" #include "mb/pg_wchar.h" @@ -47,10 +48,12 @@ #include "parser/parse_func.h" #include "parser/parse_oper.h" #include "partitioning/partdesc.h" +#include "pgstat.h" #include "rewrite/rewriteManip.h" #include "storage/lmgr.h" #include "storage/proc.h" #include "storage/procarray.h" +#include "storage/sinvaladt.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -334,7 +337,7 @@ CheckIndexCompatible(Oid oldId, * doesn't show up in the output, we know we can forget about it. */ static void -WaitForOlderSnapshots(TransactionId limitXmin) +WaitForOlderSnapshots(TransactionId limitXmin, bool progress) { int n_old_snapshots; int i; @@ -343,6 +346,8 @@ WaitForOlderSnapshots(TransactionId limitXmin) old_snapshots = GetCurrentVirtualXIDs(limitXmin, true, false, PROC_IS_AUTOVACUUM | PROC_IN_VACUUM, &n_old_snapshots); + if (progress) + pgstat_progress_update_param(PROGRESS_WAITFOR_TOTAL, n_old_snapshots); for (i = 0; i < n_old_snapshots; i++) { @@ -378,7 +383,19 @@ WaitForOlderSnapshots(TransactionId limitXmin) } if (VirtualTransactionIdIsValid(old_snapshots[i])) + { + if (progress) + { + PGPROC *holder = BackendIdGetProc(old_snapshots[i].backendId); + + pgstat_progress_update_param(PROGRESS_WAITFOR_CURRENT_PID, + holder->pid); + } VirtualXactLock(old_snapshots[i], true); + } + + if (progress) + pgstat_progress_update_param(PROGRESS_WAITFOR_DONE, i + 1); } } @@ -452,6 +469,15 @@ DefineIndex(Oid relationId, Snapshot snapshot; int i; + + /* + * Start progress report. If we're building a partition, this was already + * done. + */ + if (!OidIsValid(parentIndexId)) + pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, + relationId); + /* * count key attributes in index */ @@ -668,6 +694,9 @@ DefineIndex(Oid relationId, accessMethodId = accessMethodForm->oid; amRoutine = GetIndexAmRoutine(accessMethodForm->amhandler); + pgstat_progress_update_param(PROGRESS_CREATEIDX_ACCESS_METHOD_OID, + accessMethodId); + if (stmt->unique && !amRoutine->amcanunique) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -948,6 +977,11 @@ DefineIndex(Oid relationId, if (!OidIsValid(indexRelationId)) { table_close(rel, NoLock); + + /* If this is the top-level index, we're done */ + if (!OidIsValid(parentIndexId)) + pgstat_progress_end_command(); + return address; } @@ -973,6 +1007,9 @@ DefineIndex(Oid relationId, TupleDesc parentDesc; Oid *opfamOids; + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, + nparts); + memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); @@ -1122,6 +1159,8 @@ DefineIndex(Oid relationId, skip_build, quiet); } + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); pfree(attmap); } @@ -1156,6 +1195,8 @@ DefineIndex(Oid relationId, * Indexes on partitioned tables are not themselves built, so we're * done here. */ + if (!OidIsValid(parentIndexId)) + pgstat_progress_end_command(); return address; } @@ -1163,6 +1204,11 @@ DefineIndex(Oid relationId, { /* Close the heap and we're done, in the non-concurrent case */ table_close(rel, NoLock); + + /* If this is the top-level index, we're done. */ + if (!OidIsValid(parentIndexId)) + pgstat_progress_end_command(); + return address; } @@ -1214,7 +1260,9 @@ DefineIndex(Oid relationId, * exclusive lock on our table. The lock code will detect deadlock and * error out properly. */ - WaitForLockers(heaplocktag, ShareLock); + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_WAIT_1); + WaitForLockers(heaplocktag, ShareLock, true); /* * At this moment we are sure that there are no transactions with the @@ -1255,7 +1303,9 @@ DefineIndex(Oid relationId, * We once again wait until no transaction can have the table open with * the index marked as read-only for updates. */ - WaitForLockers(heaplocktag, ShareLock); + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_WAIT_2); + WaitForLockers(heaplocktag, ShareLock, true); /* * Now take the "reference snapshot" that will be used by validate_index() @@ -1312,7 +1362,9 @@ DefineIndex(Oid relationId, * before the reference snap was taken, we have to wait out any * transactions that might have older snapshots. */ - WaitForOlderSnapshots(limitXmin); + pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PHASE_WAIT_3); + WaitForOlderSnapshots(limitXmin, true); /* * Index can now be marked valid -- update its pg_index entry @@ -1334,6 +1386,8 @@ DefineIndex(Oid relationId, */ UnlockRelationIdForSession(&heaprelid, ShareUpdateExclusiveLock); + pgstat_progress_end_command(); + return address; } @@ -2913,7 +2967,7 @@ ReindexRelationConcurrently(Oid relationOid, int options) * DefineIndex() for more details. */ - WaitForLockersMultiple(lockTags, ShareLock); + WaitForLockersMultiple(lockTags, ShareLock, false); CommitTransactionCommand(); forboth(lc, indexIds, lc2, newIndexIds) @@ -2955,7 +3009,7 @@ ReindexRelationConcurrently(Oid relationOid, int options) * for more details. */ - WaitForLockersMultiple(lockTags, ShareLock); + WaitForLockersMultiple(lockTags, ShareLock, false); CommitTransactionCommand(); foreach(lc, newIndexIds) @@ -3003,7 +3057,7 @@ ReindexRelationConcurrently(Oid relationOid, int options) * before the reference snap was taken, we have to wait out any * transactions that might have older snapshots. */ - WaitForOlderSnapshots(limitXmin); + WaitForOlderSnapshots(limitXmin, false); CommitTransactionCommand(); } @@ -3074,7 +3128,7 @@ ReindexRelationConcurrently(Oid relationOid, int options) * index_drop() for more details. */ - WaitForLockersMultiple(lockTags, AccessExclusiveLock); + WaitForLockersMultiple(lockTags, AccessExclusiveLock, false); foreach(lc, indexIds) { @@ -3096,7 +3150,7 @@ ReindexRelationConcurrently(Oid relationOid, int options) * Drop the old indexes. */ - WaitForLockersMultiple(lockTags, AccessExclusiveLock); + WaitForLockersMultiple(lockTags, AccessExclusiveLock, false); PushActiveSnapshot(GetTransactionSnapshot()); diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c index cd56dca3aef..215f1463bb1 100644 --- a/src/backend/storage/ipc/standby.c +++ b/src/backend/storage/ipc/standby.c @@ -401,7 +401,7 @@ ResolveRecoveryConflictWithLock(LOCKTAG locktag) */ VirtualTransactionId *backends; - backends = GetLockConflicts(&locktag, AccessExclusiveLock); + backends = GetLockConflicts(&locktag, AccessExclusiveLock, NULL); ResolveRecoveryConflictWithVirtualXIDs(backends, PROCSIG_RECOVERY_CONFLICT_LOCK); } diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c index e688ba81170..0b04b093782 100644 --- a/src/backend/storage/lmgr/lmgr.c +++ b/src/backend/storage/lmgr/lmgr.c @@ -19,9 +19,12 @@ #include "access/transam.h" #include "access/xact.h" #include "catalog/catalog.h" +#include "commands/progress.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/lmgr.h" #include "storage/procarray.h" +#include "storage/sinvaladt.h" #include "utils/inval.h" @@ -857,10 +860,12 @@ XactLockTableWaitErrorCb(void *arg) * after we obtained our initial list of lockers, we will not wait for them. */ void -WaitForLockersMultiple(List *locktags, LOCKMODE lockmode) +WaitForLockersMultiple(List *locktags, LOCKMODE lockmode, bool progress) { List *holders = NIL; ListCell *lc; + int total = 0; + int done = 0; /* Done if no locks to wait for */ if (list_length(locktags) == 0) @@ -870,10 +875,17 @@ WaitForLockersMultiple(List *locktags, LOCKMODE lockmode) foreach(lc, locktags) { LOCKTAG *locktag = lfirst(lc); + int count; - holders = lappend(holders, GetLockConflicts(locktag, lockmode)); + holders = lappend(holders, + GetLockConflicts(locktag, lockmode, + progress ? &count : NULL)); + total += count; } + if (progress) + pgstat_progress_update_param(PROGRESS_WAITFOR_TOTAL, total); + /* * Note: GetLockConflicts() never reports our own xid, hence we need not * check for that. Also, prepared xacts are not reported, which is fine @@ -887,10 +899,36 @@ WaitForLockersMultiple(List *locktags, LOCKMODE lockmode) while (VirtualTransactionIdIsValid(*lockholders)) { + /* + * If requested, publish who we're going to wait for. This is not + * 100% accurate if they're already gone, but we don't care. + */ + if (progress) + { + PGPROC *holder = BackendIdGetProc(lockholders->backendId); + + pgstat_progress_update_param(PROGRESS_WAITFOR_CURRENT_PID, + holder->pid); + } VirtualXactLock(*lockholders, true); lockholders++; + + if (progress) + pgstat_progress_update_param(PROGRESS_WAITFOR_DONE, ++done); } } + if (progress) + { + const int index[] = { + PROGRESS_WAITFOR_TOTAL, + PROGRESS_WAITFOR_DONE, + PROGRESS_WAITFOR_CURRENT_PID + }; + const int64 values[] = { + 0, 0, 0 + }; + pgstat_progress_update_multi_param(3, index, values); + } list_free_deep(holders); } @@ -901,12 +939,12 @@ WaitForLockersMultiple(List *locktags, LOCKMODE lockmode) * Same as WaitForLockersMultiple, for a single lock tag. */ void -WaitForLockers(LOCKTAG heaplocktag, LOCKMODE lockmode) +WaitForLockers(LOCKTAG heaplocktag, LOCKMODE lockmode, bool progress) { List *l; l = list_make1(&heaplocktag); - WaitForLockersMultiple(l, lockmode); + WaitForLockersMultiple(l, lockmode, progress); list_free(l); } diff --git a/src/backend/storage/lmgr/lock.c b/src/backend/storage/lmgr/lock.c index 78fdbd6ff88..c8958766f1e 100644 --- a/src/backend/storage/lmgr/lock.c +++ b/src/backend/storage/lmgr/lock.c @@ -2807,6 +2807,7 @@ FastPathGetRelationLockEntry(LOCALLOCK *locallock) * xacts merely awaiting such a lock are NOT reported. * * The result array is palloc'd and is terminated with an invalid VXID. + * *countp, if not null, is updated to the number of items set. * * Of course, the result could be out of date by the time it's returned, * so use of this function has to be thought about carefully. @@ -2817,7 +2818,7 @@ FastPathGetRelationLockEntry(LOCALLOCK *locallock) * uses of the result. */ VirtualTransactionId * -GetLockConflicts(const LOCKTAG *locktag, LOCKMODE lockmode) +GetLockConflicts(const LOCKTAG *locktag, LOCKMODE lockmode, int *countp) { static VirtualTransactionId *vxids; LOCKMETHODID lockmethodid = locktag->locktag_lockmethodid; @@ -2964,6 +2965,8 @@ GetLockConflicts(const LOCKTAG *locktag, LOCKMODE lockmode) LWLockRelease(partitionLock); vxids[count].backendId = InvalidBackendId; vxids[count].localTransactionId = InvalidLocalTransactionId; + if (countp) + *countp = count; return vxids; } @@ -3019,6 +3022,8 @@ GetLockConflicts(const LOCKTAG *locktag, LOCKMODE lockmode) vxids[count].backendId = InvalidBackendId; vxids[count].localTransactionId = InvalidLocalTransactionId; + if (countp) + *countp = count; return vxids; } diff --git a/src/backend/utils/adt/amutils.c b/src/backend/utils/adt/amutils.c index 060ffe501ec..e81d6cc0562 100644 --- a/src/backend/utils/adt/amutils.c +++ b/src/backend/utils/adt/amutils.c @@ -445,3 +445,26 @@ pg_index_column_has_property(PG_FUNCTION_ARGS) return indexam_property(fcinfo, propname, InvalidOid, relid, attno); } + +/* + * Return the name of the given phase, as used for progress reporting by the + * given AM. + */ +Datum +pg_indexam_progress_phasename(PG_FUNCTION_ARGS) +{ + Oid amoid = PG_GETARG_OID(0); + int32 phasenum = PG_GETARG_INT32(1); + IndexAmRoutine *routine; + char *name; + + routine = GetIndexAmRoutineByAmId(amoid, true); + if (routine == NULL || !routine->ambuildphasename) + PG_RETURN_NULL(); + + name = routine->ambuildphasename(phasenum); + if (!name) + PG_RETURN_NULL(); + + PG_RETURN_TEXT_P(CStringGetTextDatum(name)); +} diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 90a817a25c5..7c2afe64272 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -470,6 +470,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) cmdtype = PROGRESS_COMMAND_VACUUM; else if (pg_strcasecmp(cmd, "CLUSTER") == 0) cmdtype = PROGRESS_COMMAND_CLUSTER; + else if (pg_strcasecmp(cmd, "CREATE INDEX") == 0) + cmdtype = PROGRESS_COMMAND_CREATE_INDEX; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), diff --git a/src/include/access/amapi.h b/src/include/access/amapi.h index 653ddc976ba..09a7404267c 100644 --- a/src/include/access/amapi.h +++ b/src/include/access/amapi.h @@ -108,6 +108,9 @@ typedef bool (*amproperty_function) (Oid index_oid, int attno, IndexAMProperty prop, const char *propname, bool *res, bool *isnull); +/* name of phase as used in progress reporting */ +typedef char *(*ambuildphasename_function) (int64 phasenum); + /* validate definition of an opclass for this AM */ typedef bool (*amvalidate_function) (Oid opclassoid); @@ -213,6 +216,7 @@ typedef struct IndexAmRoutine amcostestimate_function amcostestimate; amoptions_function amoptions; amproperty_function amproperty; /* can be NULL */ + ambuildphasename_function ambuildphasename; /* can be NULL */ amvalidate_function amvalidate; ambeginscan_function ambeginscan; amrescan_function amrescan; diff --git a/src/include/access/genam.h b/src/include/access/genam.h index 70c7351a08c..9717183ef23 100644 --- a/src/include/access/genam.h +++ b/src/include/access/genam.h @@ -45,6 +45,7 @@ typedef struct IndexVacuumInfo { Relation index; /* the index being vacuumed */ bool analyze_only; /* ANALYZE (without any actual vacuum) */ + bool report_progress; /* emit progress.h status reports */ bool estimated_count; /* num_heap_tuples is an estimate */ int message_level; /* ereport level for progress messages */ double num_heap_tuples; /* tuples remaining in heap */ diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index a1ffa983365..fbc8134cfdb 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -671,6 +671,16 @@ typedef BTScanOpaqueData *BTScanOpaque; #define SK_BT_DESC (INDOPTION_DESC << SK_BT_INDOPTION_SHIFT) #define SK_BT_NULLS_FIRST (INDOPTION_NULLS_FIRST << SK_BT_INDOPTION_SHIFT) +/* + * Constant definition for progress reporting. Phase numbers must match + * btbuildphasename. + */ +/* PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE is 1 (see progress.h) */ +#define PROGRESS_BTREE_PHASE_INDEXBUILD_TABLESCAN 2 +#define PROGRESS_BTREE_PHASE_PERFORMSORT_1 3 +#define PROGRESS_BTREE_PHASE_PERFORMSORT_2 4 +#define PROGRESS_BTREE_PHASE_LEAF_LOAD 5 + /* * external entry points for btree, in nbtree.c */ @@ -784,6 +794,7 @@ extern bytea *btoptions(Datum reloptions, bool validate); extern bool btproperty(Oid index_oid, int attno, IndexAMProperty prop, const char *propname, bool *res, bool *isnull); +extern char *btbuildphasename(int64 phasenum); extern IndexTuple _bt_truncate(Relation rel, IndexTuple lastleft, IndexTuple firstright, BTScanInsert itup_key); extern int _bt_keep_natts_fast(Relation rel, IndexTuple lastleft, diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 4efe178ed1e..4b760c2cd75 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -507,6 +507,7 @@ typedef struct TableAmRoutine struct IndexInfo *index_nfo, bool allow_sync, bool anyvisible, + bool progress, BlockNumber start_blockno, BlockNumber end_blockno, IndexBuildCallback callback, @@ -1369,6 +1370,8 @@ table_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, * so here because the AM might reject some of the tuples for its own reasons, * such as being unable to store NULLs. * + * If 'progress', the PROGRESS_SCAN_BLOCKS_TOTAL counter is updated when + * starting the scan, and PROGRESS_SCAN_BLOCKS_DONE is updated as we go along. * * A side effect is to set indexInfo->ii_BrokenHotChain to true if we detect * any potentially broken HOT chains. Currently, we set this if there are any @@ -1382,6 +1385,7 @@ table_index_build_scan(Relation heap_rel, Relation index_rel, struct IndexInfo *index_nfo, bool allow_sync, + bool progress, IndexBuildCallback callback, void *callback_state, TableScanDesc scan) @@ -1391,6 +1395,7 @@ table_index_build_scan(Relation heap_rel, index_nfo, allow_sync, false, + progress, 0, InvalidBlockNumber, callback, @@ -1414,6 +1419,7 @@ table_index_build_range_scan(Relation heap_rel, struct IndexInfo *index_nfo, bool allow_sync, bool anyvisible, + bool progress, BlockNumber start_blockno, BlockNumber numblocks, IndexBuildCallback callback, @@ -1425,6 +1431,7 @@ table_index_build_range_scan(Relation heap_rel, index_nfo, allow_sync, anyvisible, + progress, start_blockno, numblocks, callback, diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index dfb94bfadfb..43142b67c58 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201904011 +#define CATALOG_VERSION_NO 201904021 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index eac909109c5..a7050edca09 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -924,6 +924,10 @@ proname => 'pg_index_column_has_property', provolatile => 's', prorettype => 'bool', proargtypes => 'regclass int4 text', prosrc => 'pg_index_column_has_property' }, +{ oid => '676', descr => 'return name of given index build phase', + proname => 'pg_indexam_progress_phasename', provolatile => 'i', + prorettype => 'text', proargtypes => 'oid int8', + prosrc => 'pg_indexam_progress_phasename' }, { oid => '339', proname => 'poly_same', prorettype => 'bool', @@ -5122,9 +5126,9 @@ proname => 'pg_stat_get_progress_info', prorows => '100', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'text', - proallargtypes => '{text,int4,oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8}', - proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{cmdtype,pid,datid,relid,param1,param2,param3,param4,param5,param6,param7,param8,param9,param10}', + proallargtypes => '{text,int4,oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{cmdtype,pid,datid,relid,param1,param2,param3,param4,param5,param6,param7,param8,param9,param10,param11,param12,param13,param14,param15,param16,param17,param18,param19,param20}', prosrc => 'pg_stat_get_progress_info' }, { oid => '3099', descr => 'statistics: information about currently active replication', diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index 04542d9e923..f046fa13b18 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -44,7 +44,7 @@ #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7 -/* Phases of cluster (as dvertised via PROGRESS_CLUSTER_PHASE) */ +/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */ #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1 #define PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP 2 #define PROGRESS_CLUSTER_PHASE_SORT_TUPLES 3 @@ -57,4 +57,39 @@ #define PROGRESS_CLUSTER_COMMAND_CLUSTER 1 #define PROGRESS_CLUSTER_COMMAND_VACUUM_FULL 2 +/* Progress parameters for CREATE INDEX */ +/* 3, 4 and 5 reserved for "waitfor" metrics */ +#define PROGRESS_CREATEIDX_ACCESS_METHOD_OID 8 +#define PROGRESS_CREATEIDX_PHASE 9 /* AM-agnostic phase # */ +#define PROGRESS_CREATEIDX_SUBPHASE 10 /* phase # filled by AM */ +#define PROGRESS_CREATEIDX_TUPLES_TOTAL 11 +#define PROGRESS_CREATEIDX_TUPLES_DONE 12 +#define PROGRESS_CREATEIDX_PARTITIONS_TOTAL 13 +#define PROGRESS_CREATEIDX_PARTITIONS_DONE 14 +/* 15 and 16 reserved for "block number" metrics */ + +/* Phases of CREATE INDEX (as advertised via PROGRESS_CREATEIDX_PHASE) */ +#define PROGRESS_CREATEIDX_PHASE_WAIT_1 1 +#define PROGRESS_CREATEIDX_PHASE_BUILD 2 +#define PROGRESS_CREATEIDX_PHASE_WAIT_2 3 +#define PROGRESS_CREATEIDX_PHASE_VALIDATE_IDXSCAN 4 +#define PROGRESS_CREATEIDX_PHASE_VALIDATE_SORT 5 +#define PROGRESS_CREATEIDX_PHASE_VALIDATE_TABLESCAN 6 +#define PROGRESS_CREATEIDX_PHASE_WAIT_3 7 + +/* + * Subphases of CREATE INDEX, for index_build. + */ +#define PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE 1 +/* Additional phases are defined by each AM */ + +/* Lock holder wait counts */ +#define PROGRESS_WAITFOR_TOTAL 3 +#define PROGRESS_WAITFOR_DONE 4 +#define PROGRESS_WAITFOR_CURRENT_PID 5 + +/* Block numbers in a generic relation scan */ +#define PROGRESS_SCAN_BLOCKS_TOTAL 15 +#define PROGRESS_SCAN_BLOCKS_DONE 16 + #endif diff --git a/src/include/pgstat.h b/src/include/pgstat.h index c080fa6388f..53d4a9c4319 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -951,10 +951,11 @@ typedef enum ProgressCommandType { PROGRESS_COMMAND_INVALID, PROGRESS_COMMAND_VACUUM, - PROGRESS_COMMAND_CLUSTER + PROGRESS_COMMAND_CLUSTER, + PROGRESS_COMMAND_CREATE_INDEX } ProgressCommandType; -#define PGSTAT_NUM_PROGRESS_PARAM 10 +#define PGSTAT_NUM_PROGRESS_PARAM 20 /* ---------- * Shared-memory data structures diff --git a/src/include/storage/lmgr.h b/src/include/storage/lmgr.h index 3d705faba5c..4f2872de35f 100644 --- a/src/include/storage/lmgr.h +++ b/src/include/storage/lmgr.h @@ -78,8 +78,8 @@ extern void XactLockTableWait(TransactionId xid, Relation rel, extern bool ConditionalXactLockTableWait(TransactionId xid); /* Lock VXIDs, specified by conflicting locktags */ -extern void WaitForLockers(LOCKTAG heaplocktag, LOCKMODE lockmode); -extern void WaitForLockersMultiple(List *locktags, LOCKMODE lockmode); +extern void WaitForLockers(LOCKTAG heaplocktag, LOCKMODE lockmode, bool progress); +extern void WaitForLockersMultiple(List *locktags, LOCKMODE lockmode, bool progress); /* Lock an XID for tuple insertion (used to wait for an insertion to finish) */ extern uint32 SpeculativeInsertionLockAcquire(TransactionId xid); diff --git a/src/include/storage/lock.h b/src/include/storage/lock.h index badf7fd682b..048947c50d4 100644 --- a/src/include/storage/lock.h +++ b/src/include/storage/lock.h @@ -544,7 +544,7 @@ extern bool LockHeldByMe(const LOCKTAG *locktag, LOCKMODE lockmode); extern bool LockHasWaiters(const LOCKTAG *locktag, LOCKMODE lockmode, bool sessionLock); extern VirtualTransactionId *GetLockConflicts(const LOCKTAG *locktag, - LOCKMODE lockmode); + LOCKMODE lockmode, int *countp); extern void AtPrepare_Locks(void); extern void PostPrepare_Locks(TransactionId xid); extern int LockCheckConflicts(LockMethod lockMethodTable, diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d5f309fbfbe..51d7a150cfb 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1856,7 +1856,33 @@ pg_stat_progress_cluster| SELECT s.pid, s.param6 AS heap_blks_total, s.param7 AS heap_blks_scanned, s.param8 AS index_rebuild_count - FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10) + FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) + LEFT JOIN pg_database d ON ((s.datid = d.oid))); +pg_stat_progress_create_index| SELECT s.pid, + s.datid, + d.datname, + s.relid, + CASE s.param10 + WHEN 0 THEN 'initializing'::text + WHEN 1 THEN 'waiting for old snapshots'::text + WHEN 2 THEN ('building index'::text || COALESCE((': '::text || pg_indexam_progress_phasename((s.param9)::oid, s.param11)), ''::text)) + WHEN 3 THEN 'waiting for writer snapshots'::text + WHEN 4 THEN 'index validation: scan index'::text + WHEN 5 THEN 'index validation: sort index scan results'::text + WHEN 6 THEN 'index validation: scan heap'::text + WHEN 7 THEN 'waiting for reader snapshots'::text + ELSE NULL::text + END AS phase, + s.param4 AS lockers_total, + s.param5 AS lockers_done, + s.param6 AS current_locker_pid, + s.param16 AS blocks_total, + s.param17 AS blocks_done, + s.param12 AS tuples_total, + s.param13 AS tuples_done, + s.param14 AS partitions_total, + s.param15 AS partitions_done + FROM (pg_stat_get_progress_info('CREATE INDEX'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_progress_vacuum| SELECT s.pid, s.datid, @@ -1878,7 +1904,7 @@ pg_stat_progress_vacuum| SELECT s.pid, s.param5 AS index_vacuum_count, s.param6 AS max_dead_tuples, s.param7 AS num_dead_tuples - FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10) + FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_replication| SELECT s.pid, s.usesysid,