pg_basebackup: Add --nosync option

This is useful for testing, similar to initdb's --nosync.

From: Michael Paquier <[email protected]>

diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index fe557ed002f2eb1edf37f00852ea079715d68bbe..55e913f70de511baf6ab277b48df80e23dd7a199 100644 (file)
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -438,6 +438,21 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-N</option></term>
+      <term><option>--nosync</option></term>
+      <listitem>
+       <para>
+        By default, <command>pg_basebackup</command> will wait for all files
+        to be written safely to disk.  This option causes
+        <command>pg_basebackup</command> to return without waiting, which is
+        faster, but means that a subsequent operating system crash can leave
+        the base backup corrupt.  Generally, this option is useful for testing
+        but should not be used when creating a production installation.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-v</option></term>
       <term><option>--verbose</option></term>

diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index cd7d095103df953f1d1e68ed63be42d83ee84d47..0f5d9d6a87f57358d053994a22700a0b76f4cdab 100644 (file)
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -69,6 +69,7 @@ static bool includewal = false;
 static bool streamwal = false;
 static bool fastcheckpoint = false;
 static bool writerecoveryconf = false;
+static bool do_sync = true;
 static int standby_message_timeout = 10 * 1000;        /* 10 sec = default */
 static pg_time_t last_progress_report = 0;
 static int32 maxrate = 0;      /* no limit by default */
@@ -329,6 +330,7 @@ usage(void)
             "                         set fast or spread checkpointing\n"));
    printf(_("  -l, --label=LABEL      set backup label\n"));
    printf(_("  -n, --noclean          do not clean up after errors\n"));
+   printf(_("  -N, --nosync           do not wait for changes to be written safely to disk\n"));
    printf(_("  -P, --progress         show progress information\n"));
    printf(_("  -v, --verbose          output verbose messages\n"));
    printf(_("  -V, --version          output version information, then exit\n"));
@@ -460,6 +462,7 @@ LogStreamerMain(logstreamer_param *param)
    stream.stream_stop = reached_end_position;
    stream.standby_message_timeout = standby_message_timeout;
    stream.synchronous = false;
+   stream.do_sync = do_sync;
    stream.mark_done = true;
    stream.basedir = param->xlogdir;
    stream.partial_suffix = NULL;
@@ -1199,7 +1202,7 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
        PQfreemem(copybuf);
 
    /* sync the resulting tar file, errors are not considered fatal */
-   if (strcmp(basedir, "-") != 0)
+   if (do_sync && strcmp(basedir, "-") != 0)
        (void) fsync_fname(filename, false, progname);
 }
 
@@ -1967,14 +1970,17 @@ BaseBackup(void)
     * all the data of the base directory is synced, taking into account
     * all the tablespaces. Errors are not considered fatal.
     */
-   if (format == 't')
+   if (do_sync)
    {
-       if (strcmp(basedir, "-") != 0)
-           (void) fsync_fname(basedir, true, progname);
-   }
-   else
-   {
-       (void) fsync_pgdata(basedir, progname);
+       if (format == 't')
+       {
+           if (strcmp(basedir, "-") != 0)
+               (void) fsync_fname(basedir, true, progname);
+       }
+       else
+       {
+           (void) fsync_pgdata(basedir, progname);
+       }
    }
 
    if (verbose)
@@ -2001,6 +2007,7 @@ main(int argc, char **argv)
        {"compress", required_argument, NULL, 'Z'},
        {"label", required_argument, NULL, 'l'},
        {"noclean", no_argument, NULL, 'n'},
+       {"nosync", no_argument, NULL, 'N'},
        {"dbname", required_argument, NULL, 'd'},
        {"host", required_argument, NULL, 'h'},
        {"port", required_argument, NULL, 'p'},
@@ -2037,7 +2044,7 @@ main(int argc, char **argv)
 
    atexit(cleanup_directories_atexit);
 
-   while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nzZ:d:c:h:p:U:s:S:wWvP",
+   while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nNzZ:d:c:h:p:U:s:S:wWvP",
                            long_options, &option_index)) != -1)
    {
        switch (c)
@@ -2115,6 +2122,9 @@ main(int argc, char **argv)
            case 'n':
                noclean = true;
                break;
+           case 'N':
+               do_sync = false;
+               break;
            case 'z':
 #ifdef HAVE_LIBZ
                compresslevel = Z_DEFAULT_COMPRESSION;

diff --git a/src/bin/pg_basebackup/pg_receivexlog.c b/src/bin/pg_basebackup/pg_receivexlog.c
index 7f7ee9dc9baeb6e0986e3ee06f5146bb195b04d1..a58a251a59f82086d22e577965245ddf3218abb1 100644 (file)
--- a/src/bin/pg_basebackup/pg_receivexlog.c
+++ b/src/bin/pg_basebackup/pg_receivexlog.c
@@ -336,6 +336,7 @@ StreamLog(void)
    stream.stream_stop = stop_streaming;
    stream.standby_message_timeout = standby_message_timeout;
    stream.synchronous = synchronous;
+   stream.do_sync = true;
    stream.mark_done = false;
    stream.basedir = basedir;
    stream.partial_suffix = ".partial";

diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c
index 6b78a60f2780bf9a4cbafa80e5a64215dd57cdfc..8f29d19114469e89e4400c09c1cb208e57cc6756 100644 (file)
--- a/src/bin/pg_basebackup/receivelog.c
+++ b/src/bin/pg_basebackup/receivelog.c
@@ -41,8 +41,8 @@ static PGresult *HandleCopyStream(PGconn *conn, StreamCtl *stream,
                 XLogRecPtr *stoppos);
 static int CopyStreamPoll(PGconn *conn, long timeout_ms);
 static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
-static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
-                   XLogRecPtr blockpos, int64 *last_status);
+static bool ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf,
+                   int len, XLogRecPtr blockpos, int64 *last_status);
 static bool ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len,
                   XLogRecPtr *blockpos);
 static PGresult *HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf,
@@ -56,7 +56,7 @@ static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
                         uint32 *timeline);
 
 static bool
-mark_file_as_archived(const char *basedir, const char *fname)
+mark_file_as_archived(const char *basedir, const char *fname, bool do_sync)
 {
    int         fd;
    static char tmppath[MAXPGPATH];
@@ -74,10 +74,10 @@ mark_file_as_archived(const char *basedir, const char *fname)
 
    close(fd);
 
-   if (fsync_fname(tmppath, false, progname) != 0)
+   if (do_sync && fsync_fname(tmppath, false, progname) != 0)
        return false;
 
-   if (fsync_parent_path(tmppath, progname) != 0)
+   if (do_sync && fsync_parent_path(tmppath, progname) != 0)
        return false;
 
    return true;
@@ -134,9 +134,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
         * fsync, in case of a previous crash between padding and fsyncing the
         * file.
         */
-       if (fsync_fname(fn, false, progname) != 0)
+       if (stream->do_sync && fsync_fname(fn, false, progname) != 0)
            return false;
-       if (fsync_parent_path(fn, progname) != 0)
+       if (stream->do_sync && fsync_parent_path(fn, progname) != 0)
            return false;
 
        return true;
@@ -173,9 +173,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
     * using synchronous mode, where the file is modified and fsynced
     * in-place, without a directory fsync.
     */
-   if (fsync_fname(fn, false, progname) != 0)
+   if (stream->do_sync && fsync_fname(fn, false, progname) != 0)
        return false;
-   if (fsync_parent_path(fn, progname) != 0)
+   if (stream->do_sync && fsync_parent_path(fn, progname) != 0)
        return false;
 
    if (lseek(f, SEEK_SET, 0) != 0)
@@ -212,7 +212,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
        return false;
    }
 
-   if (fsync(walfile) != 0)
+   if (stream->do_sync && fsync(walfile) != 0)
    {
        fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
                progname, current_walfile_name, strerror(errno));
@@ -258,7 +258,8 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
    if (currpos == XLOG_SEG_SIZE && stream->mark_done)
    {
        /* writes error message if failed */
-       if (!mark_file_as_archived(stream->basedir, current_walfile_name))
+       if (!mark_file_as_archived(stream->basedir, current_walfile_name,
+                                  stream->do_sync))
            return false;
    }
 
@@ -378,7 +379,8 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content)
    if (stream->mark_done)
    {
        /* writes error message if failed */
-       if (!mark_file_as_archived(stream->basedir, histfname))
+       if (!mark_file_as_archived(stream->basedir, histfname,
+                                  stream->do_sync))
            return false;
    }
 
@@ -836,7 +838,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream,
         */
        if (stream->synchronous && lastFlushPosition < blockpos && walfile != -1)
        {
-           if (fsync(walfile) != 0)
+           if (stream->do_sync && fsync(walfile) != 0)
            {
                fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
                        progname, current_walfile_name, strerror(errno));
@@ -890,7 +892,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream,
            /* Check the message type. */
            if (copybuf[0] == 'k')
            {
-               if (!ProcessKeepaliveMsg(conn, copybuf, r, blockpos,
+               if (!ProcessKeepaliveMsg(conn, stream, copybuf, r, blockpos,
                                         &last_status))
                    goto error;
            }
@@ -1043,7 +1045,7 @@ CopyStreamReceive(PGconn *conn, long timeout, char **buffer)
  * Process the keepalive message.
  */
 static bool
-ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
+ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len,
                    XLogRecPtr blockpos, int64 *last_status)
 {
    int         pos;
@@ -1079,7 +1081,7 @@ ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
             * data has been successfully replicated or not, at the normal
             * shutdown of the server.
             */
-           if (fsync(walfile) != 0)
+           if (stream->do_sync && fsync(walfile) != 0)
            {
                fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
                        progname, current_walfile_name, strerror(errno));

diff --git a/src/bin/pg_basebackup/receivelog.h b/src/bin/pg_basebackup/receivelog.h
index 554ff8b5b28610df1385f3d2342b57f9c485a91d..7a3bbc50800d1f540ed93b311c36b15b7293e92c 100644 (file)
--- a/src/bin/pg_basebackup/receivelog.h
+++ b/src/bin/pg_basebackup/receivelog.h
@@ -34,8 +34,10 @@ typedef struct StreamCtl
                                 * timeline */
    int         standby_message_timeout;        /* Send status messages this
                                                 * often */
-   bool        synchronous;    /* Flush data on write */
+   bool        synchronous;    /* Flush immediately WAL data on write */
    bool        mark_done;      /* Mark segment as done in generated archive */
+   bool        do_sync;        /* Flush to disk to ensure consistent state
+                                * of data */
 
    stream_stop_callback stream_stop;   /* Stop streaming when returns true */