Merge branch 'pg_snap' of https://github.com/postgrespro/snapfs into pg_snap

Author: rzharkov

LICENSE

@@ -0,0 +1,22 @@
+Snapfs is released under the PostgreSQL License, a liberal Open Source
+license, similar to the BSD or MIT licenses.
+
+Copyright (c) 2018, Postgres Professional
+Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
+Portions Copyright (c) 1994, The Regents of the University of California
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose, without fee, and without a written agreement is
+hereby granted, provided that the above copyright notice and this paragraph and
+the following two paragraphs appear in all copies.
+
+IN NO EVENT SHALL POSTGRES PROFESSIONAL BE LIABLE TO ANY PARTY FOR DIRECT,
+INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
+ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF POSTGRES
+PROFESSIONAL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+POSTGRES PROFESSIONAL SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND
+POSTGRES PROFESSIONAL HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT,
+UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

src/backend/replication/basebackup.c

@@ -37,6 +37,7 @@
 #include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/reinit.h"
+#include "storage/snapfs.h"
 #include "utils/builtins.h"
 #include "utils/ps_status.h"
 #include "utils/relcache.h"
@@ -52,6 +53,7 @@ typedef struct
 	bool		includewal;
 	uint32		maxrate;
 	bool		sendtblspcmapfile;
+	SnapshotId  snapshot;
 } basebackup_options;
 
 
@@ -74,6 +76,8 @@ static int	compareWalFileNames(const void *a, const void *b);
 static void throttle(size_t increment);
 static bool is_checksummed_file(const char *fullpath, const char *filename);
 
+#define BACKUP_ALL_SNAPSHOTS ((SnapshotId)~0)
+
 /* Was the backup currently in-progress initiated in recovery mode? */
 static bool backup_started_in_recovery = false;
 
@@ -234,6 +238,7 @@ perform_base_backup(basebackup_options *opt)
 	tblspc_map_file = makeStringInfo();
 
 	total_checksum_failures = 0;
+	sfs_backend_snapshot = opt->snapshot;
 
 	startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
 								  labelfile, &tablespaces,
@@ -643,8 +648,11 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 	bool		o_maxrate = false;
 	bool		o_tablespace_map = false;
 	bool		o_noverify_checksums = false;
+	bool		o_snapshot = false;
 
 	MemSet(opt, 0, sizeof(*opt));
+	opt->snapshot = BACKUP_ALL_SNAPSHOTS;
+
 	foreach(lopt, options)
 	{
 		DefElem    *defel = (DefElem *) lfirst(lopt);
@@ -658,6 +666,15 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 			opt->label = strVal(defel->arg);
 			o_label = true;
 		}
+		else if (strcmp(defel->defname, "snapshot") == 0)
+		{
+			if (o_snapshot)
+				ereport(ERROR,
+						(errcode(ERRCODE_SYNTAX_ERROR),
+						 errmsg("duplicate option \"%s\"", defel->defname)));
+			opt->snapshot = intVal(defel->arg);
+			o_snapshot = true;
+		}
 		else if (strcmp(defel->defname, "progress") == 0)
 		{
 			if (o_progress)
@@ -764,8 +781,10 @@ SendBaseBackup(BaseBackupCmd *cmd)
 				 opt.label);
 		set_ps_display(activitymsg, false);
 	}
-
+	sfs_basebackup = true;
 	perform_base_backup(&opt);
+	sfs_basebackup = false;
+	sfs_backend_snapshot = SFS_INVALID_SNAPSHOT;
 }
 
 static void
@@ -1140,6 +1159,10 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
 			continue;
 		}
 
+		/* Do not backup snapfs files */
+		if (sfs_backend_snapshot != BACKUP_ALL_SNAPSHOTS && is_snapfs_file(de->d_name))
+			continue;
+
 		snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
 
 		/* Skip pg_control here to back up it last */
@@ -1373,6 +1396,13 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
 	int			segmentno = 0;
 	char	   *segmentpath;
 	bool		verify_checksum = false;
+	SnapshotId  current_snapshot;
+	File        file = -1;
+
+	current_snapshot = sfs_backend_snapshot != SFS_INVALID_SNAPSHOT && sfs_backend_snapshot != BACKUP_ALL_SNAPSHOTS
+		? sfs_backend_snapshot : ControlFile->active_snapshot;
+	if (current_snapshot != SFS_INVALID_SNAPSHOT)
+		file = PathNameOpenFile(readfilename, O_RDONLY|PG_BINARY);
 
 	fp = AllocateFile(readfilename, "rb");
 	if (fp == NULL)
@@ -1523,6 +1553,34 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
 			}
 		}
 
+		if (current_snapshot != SFS_INVALID_SNAPSHOT)
+		{
+			for (i = 0; i < cnt / BLCKSZ; i++)
+			{
+				int rc;
+				page = buf + BLCKSZ * i;
+				rc = FileRead(file, page, BLCKSZ, WAIT_EVENT_DATA_FILE_READ);
+				if (rc < 0)
+					ereport(ERROR,
+							(errmsg("base backup could not read snapshot file, aborting backup")));
+			}
+		}
+
+		/* Remove information about snapshots from control file */
+		if (SFS_KEEPING_SNAPSHOT()
+			&& sfs_backend_snapshot != BACKUP_ALL_SNAPSHOTS
+			&& strcmp(readfilename, XLOG_CONTROL_FILE) == 0)
+		{
+			ControlFileData* ctl = (ControlFileData*)buf;
+			ctl->oldest_snapshot = 1;
+			ctl->recent_snapshot = SFS_INVALID_SNAPSHOT;
+			ctl->active_snapshot = SFS_INVALID_SNAPSHOT;
+			INIT_CRC32C(ctl->crc);
+			COMP_CRC32C(ctl->crc,
+						(char *) ctl,
+						offsetof(ControlFileData, crc));
+			FIN_CRC32C(ctl->crc);
+		}
 		/* Send the chunk as a CopyData message */
 		if (pq_putmessage('d', buf, cnt))
 			ereport(ERROR,
@@ -1541,6 +1599,8 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf
 			break;
 		}
 	}
+	if (file >= 0)
+		FileClose(file);
 
 	/* If the file was truncated while we were sending it, pad it with zeros */
 	if (len < statbuf->st_size)

src/backend/replication/repl_gram.y

@@ -70,6 +70,7 @@ static SQLCmd *make_sqlcmd(void);
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
 %token K_LABEL
+%token K_SNAPSHOT
 %token K_PROGRESS
 %token K_FAST
 %token K_WAIT
@@ -179,6 +180,11 @@ base_backup_opt:
 				  $$ = makeDefElem("label",
 								   (Node *)makeString($2), -1);
 				}
+			| K_SNAPSHOT UCONST
+				{
+				  $$ = makeDefElem("snapshot",
+								   (Node *)makeInteger($2), -1);
+				}
 			| K_PROGRESS
 				{
 				  $$ = makeDefElem("progress",

src/backend/replication/repl_scanner.l

@@ -87,6 +87,7 @@ FAST			{ return K_FAST; }
 IDENTIFY_SYSTEM		{ return K_IDENTIFY_SYSTEM; }
 SHOW		{ return K_SHOW; }
 LABEL			{ return K_LABEL; }
+SNAPSHOT		{ return K_SNAPSHOT; }
 NOWAIT			{ return K_NOWAIT; }
 PROGRESS			{ return K_PROGRESS; }
 MAX_RATE		{ return K_MAX_RATE; }

src/backend/storage/file/fd.c

@@ -2054,9 +2054,24 @@ FileRead(File file, char *buffer, int amount, uint32 wait_event_info)
 					elog(ERROR, "[SFS] Field to read snapshot file: %m");
 
 				pgstat_report_wait_end();
+				if (sfs_basebackup)
+				{
+					/* We should move file position in case of base backup to avoid explicit seeks */
+					vfdP->seekPos += amount;
+				}
 				return amount;
 			}
 		}
+		/*
+		 * In case of performing pg_basebackup of particular snapshot
+		 * we are interested only in reading original image of the page.
+		 */
+		if (sfs_basebackup)
+		{
+			/* We should move file position in case of base backup to avoid explicit seeks */
+			vfdP->seekPos += amount;
+			return 0;
+		}
 	}
 retry:
 	pgstat_report_wait_start(wait_event_info);
@@ -2174,8 +2189,6 @@ FileWrite(File file, char *buffer, int amount, uint32 wait_event_info)
 			char orig_block[BLCKSZ];
 			int rc;
 
-			vfdP->snap_map->offs[block_no] = snap_offs + 1;
-
 			rc = sfs_read_file(vfdP->fd, orig_block, BLCKSZ);
 			if (rc < 0)
 				elog(ERROR, "[SFS] Could not read file: %m");
@@ -2190,6 +2203,9 @@ FileWrite(File file, char *buffer, int amount, uint32 wait_event_info)
 
 			if (!sfs_write_file(vfdP->snap_fd, orig_block, BLCKSZ))
 				elog(ERROR, "[SFS] Could not write file: %m");
+
+			/* Update map only after saving original page */
+			vfdP->snap_map->offs[block_no] = snap_offs + 1;
 		}
 	}
 
@@ -3450,6 +3466,13 @@ RemovePgTempRelationFilesInDbspace(const char *dbspacedirname)
 	FreeDir(dbspace_dir);
 }
 
+/* <file>.snap.SNAPSHOT or <file>.snapmap.SNAPSHOT */ 
+bool
+is_snapfs_file(const char* name)
+{
+	return strstr(name, ".snap") != NULL;
+}
+
 /* t<digits>_<digits>, or t<digits>_<digits>_<forkname> */
 bool
 looks_like_temp_rel_name(const char *name)

src/backend/storage/file/snapfs.c

@@ -41,6 +41,7 @@
 
 SnapshotId sfs_backend_snapshot = SFS_INVALID_SNAPSHOT;
 SnapshotId sfs_active_snapshot;
+bool sfs_basebackup;
 
 int
 sfs_msync(SnapshotMap * map)

src/bin/pg_basebackup/pg_basebackup.c

@@ -99,6 +99,7 @@ static bool temp_replication_slot = true;
 static bool create_slot = false;
 static bool no_slot = false;
 static bool verify_checksums = true;
+static char *snapshot = NULL;
 
 static bool success = false;
 static bool made_new_pgdata = false;
@@ -366,6 +367,7 @@ usage(void)
 	printf(_("  -v, --verbose          output verbose messages\n"));
 	printf(_("  -V, --version          output version information, then exit\n"));
 	printf(_("      --no-slot          prevent creation of temporary replication slot\n"));
+	printf(_("      --snapshot=SNAPID  snapfs snapshot\n"));
 	printf(_("      --no-verify-checksums\n"
 			 "                         do not verify checksums\n"));
 	printf(_("  -?, --help             show this help, then exit\n"));
@@ -1814,15 +1816,17 @@ BaseBackup(void)
 	}
 
 	basebkp =
-		psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s",
+		psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s %s %s",
 				 escaped_label,
 				 showprogress ? "PROGRESS" : "",
 				 includewal == FETCH_WAL ? "WAL" : "",
 				 fastcheckpoint ? "FAST" : "",
 				 includewal == NO_WAL ? "" : "NOWAIT",
 				 maxrate_clause ? maxrate_clause : "",
 				 format == 't' ? "TABLESPACE_MAP" : "",
-				 verify_checksums ? "" : "NOVERIFY_CHECKSUMS");
+				 verify_checksums ? "" : "NOVERIFY_CHECKSUMS",
+				 snapshot ? "SNAPSHOT" : "",
+				 snapshot ? snapshot : "");
 
 	if (PQsendQuery(conn, basebkp) == 0)
 	{
@@ -2163,6 +2167,7 @@ main(int argc, char **argv)
 		{"waldir", required_argument, NULL, 1},
 		{"no-slot", no_argument, NULL, 2},
 		{"no-verify-checksums", no_argument, NULL, 3},
+		{"snapshot", required_argument, NULL, 4},
 		{NULL, 0, NULL, 0}
 	};
 	int			c;
@@ -2334,6 +2339,9 @@ main(int argc, char **argv)
 			case 3:
 				verify_checksums = false;
 				break;
+			case 4:
+				snapshot = pg_strdup(optarg);
+				break;
 			default:
 
 				/*

src/include/storage/fd.h

@@ -126,6 +126,7 @@ extern void AtEOXact_Files(bool isCommit);
 extern void AtEOSubXact_Files(bool isCommit, SubTransactionId mySubid,
 				  SubTransactionId parentSubid);
 extern void RemovePgTempFiles(void);
+extern bool is_snapfs_file(const char *name);
 extern bool looks_like_temp_rel_name(const char *name);
 
 extern int	pg_fsync(int fd);

src/include/storage/snapfs.h

@@ -74,6 +74,7 @@ typedef struct SnapshotMap
 
 extern SnapshotId sfs_backend_snapshot;
 extern SnapshotId sfs_active_snapshot;
+extern bool sfs_basebackup;
 
 /* Memory mapping function */
 extern int sfs_munmap(SnapshotMap * map);