Merge branch 'master' into issue_90

Author: gsmolk

Documentation.md

@@ -110,7 +110,7 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree
 
 ## Documentation
 
-Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup).
+Currently the latest documentation can be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md) and [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup).
 
 ## Licence
 

README.md

@@ -344,6 +344,9 @@ do_backup_instance(PGconn *backup_conn)
 			dir_list_file(backup_files_list, parray_get(external_dirs, i),
 						  false, true, false, i+1, FIO_DB_HOST);
 
+	/* close ssh session in main thread */
+	fio_disconnect();
+
 	/* Sanity check for backup_files_list, thank you, Windows:
 	 * https://github.com/postgrespro/pg_probackup/issues/48
 	 */
@@ -512,6 +515,9 @@ do_backup_instance(PGconn *backup_conn)
 		parray_free(prev_backup_filelist);
 	}
 
+	/* Notify end of backup */
+	pg_stop_backup(&current, pg_startbackup_conn);
+
 	/* In case of backup from replica >= 9.6 we must fix minRecPoint,
 	 * First we must find pg_control in backup_files_list.
 	 */
@@ -532,13 +538,16 @@ do_backup_instance(PGconn *backup_conn)
 				break;
 			}
 		}
-	}
 
-	/* Notify end of backup */
-	pg_stop_backup(&current, pg_startbackup_conn);
+		if (!pg_control)
+			elog(ERROR, "Failed to find file \"%s\" in backup filelist.",
+							pg_control_path);
 
-	if (current.from_replica && !exclusive_backup)
 		set_min_recovery_point(pg_control, database_path, current.stop_lsn);
+	}
+
+	/* close ssh session in main thread */
+	fio_disconnect();
 
 	/* Add archived xlog files into the list of files of this backup */
 	if (stream_wal)
@@ -2143,6 +2152,9 @@ backup_files(void *arg)
 			elog(WARNING, "unexpected file type %d", buf.st_mode);
 	}
 
+	/* ssh connection to longer needed */
+	fio_disconnect();
+
 	/* Close connection */
 	if (arguments->conn_arg.conn)
 		pgut_disconnect(arguments->conn_arg.conn);

src/backup.c

@@ -356,7 +356,7 @@ prepare_page(ConnectionArgs *arguments,
 			((strict && !is_ptrack_support) || !strict))
 		{
 			/* show this message for checkdb or backup without ptrack support */
-			elog(WARNING, "CORRUPTION in file %s, block %u",
+			elog(WARNING, "Corruption detected in file \"%s\", block %u",
 						file->path, blknum);
 		}
 
@@ -585,10 +585,7 @@ backup_data_file(backup_files_arg* arguments,
 	}
 
 	if (file->size % BLCKSZ != 0)
-	{
-		fio_fclose(in);
-		elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size);
-	}
+		elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size);
 
 	/*
 	 * Compute expected number of blocks in the file.
@@ -625,7 +622,7 @@ backup_data_file(backup_files_arg* arguments,
 			if (rc == PAGE_CHECKSUM_MISMATCH && is_ptrack_support)
 				goto RetryUsingPtrack;
 			if (rc < 0)
-				elog(ERROR, "Failed to read file %s: %s",
+				elog(ERROR, "Failed to read file \"%s\": %s",
 					 file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc));
 			n_blocks_read = rc;
 		}
@@ -1212,10 +1209,7 @@ check_data_file(ConnectionArgs *arguments,
 	}
 
 	if (file->size % BLCKSZ != 0)
-	{
-		fclose(in);
-		elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size);
-	}
+		elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size);
 
 	/*
 	 * Compute expected number of blocks in the file.

src/data.c

@@ -209,7 +209,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 	time_t		days_threshold = 0;
 
 	/* For fancy reporting */
-	float		actual_window = 0;
+	uint32		actual_window = 0;
 
 	/* Get current time */
 	current_time = time(NULL);
@@ -252,7 +252,9 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 			cur_full_backup_num++;
 		}
 
-		/* Check if backup in needed by retention policy */
+		/* Check if backup in needed by retention policy
+		 * TODO: consider that ERROR backup most likely to have recovery_time == 0
+		 */
 		if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) &&
 			(instance_config.retention_redundancy  <= (n_full_backups - cur_full_backup_num)))
 		{
@@ -324,7 +326,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 	}
 
 	/* Message about retention state of backups
-	 * TODO: Float is ugly, rewrite somehow.
+	 * TODO: message is ugly, rewrite it to something like show table in stdout.
 	 */
 
 	cur_full_backup_num = 1;
@@ -340,9 +342,9 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 		if (backup->recovery_time == 0)
 			actual_window = 0;
 		else
-			actual_window = ((float)current_time - (float)backup->recovery_time)/(60 * 60 * 24);
+			actual_window = (current_time - backup->recovery_time)/(60 * 60 * 24);
 
-		elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %.2fd/%ud. %s",
+		elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s",
 				base36enc(backup->start_time),
 				pgBackupGetBackupMode(backup),
 				status2str(backup->status),
@@ -801,10 +803,13 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
 int
 do_delete_instance(void)
 {
-	parray	   *backup_list;
-	int i;
+	parray		*backup_list;
+	parray		*xlog_files_list;
+	int 		i;
+	int 		rc;
 	char		instance_config_path[MAXPGPATH];
 
+
 	/* Delete all backups. */
 	backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
 
@@ -821,23 +826,40 @@ do_delete_instance(void)
 	parray_free(backup_list);
 
 	/* Delete all wal files. */
-	delete_walfiles(InvalidXLogRecPtr, 0, instance_config.xlog_seg_size);
+	xlog_files_list = parray_new();
+	dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
+
+	for (i = 0; i < parray_num(xlog_files_list); i++)
+	{
+		pgFile	   *wal_file = (pgFile *) parray_get(xlog_files_list, i);
+		if (S_ISREG(wal_file->mode))
+		{
+			rc = unlink(wal_file->path);
+			if (rc != 0)
+				elog(WARNING, "Failed to remove file \"%s\": %s",
+					 wal_file->path, strerror(errno));
+		}
+	}
+
+	/* Cleanup */
+	parray_walk(xlog_files_list, pgFileFree);
+	parray_free(xlog_files_list);
 
 	/* Delete backup instance config file */
 	join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
 	if (remove(instance_config_path))
 	{
-		elog(ERROR, "can't remove \"%s\": %s", instance_config_path,
+		elog(ERROR, "Can't remove \"%s\": %s", instance_config_path,
 			strerror(errno));
 	}
 
 	/* Delete instance root directories */
 	if (rmdir(backup_instance_path) != 0)
-		elog(ERROR, "can't remove \"%s\": %s", backup_instance_path,
+		elog(ERROR, "Can't remove \"%s\": %s", backup_instance_path,
 			strerror(errno));
 
 	if (rmdir(arclog_path) != 0)
-		elog(ERROR, "can't remove \"%s\": %s", arclog_path,
+		elog(ERROR, "Can't remove \"%s\": %s", arclog_path,
 			strerror(errno));
 
 	elog(INFO, "Instance '%s' successfully deleted", instance_name);

src/delete.c

@@ -122,7 +122,7 @@ static int BlackListCompare(const void *str1, const void *str2);
 
 static char dir_check_file(pgFile *file);
 static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
-								   bool omit_symlink, parray *black_list,
+								   bool follow_symlink, parray *black_list,
 								   int external_dir_num, fio_location location);
 static void opt_path_map(ConfigOption *opt, const char *arg,
 						 TablespaceList *list, const char *type);
@@ -159,14 +159,14 @@ dir_create_dir(const char *dir, mode_t mode)
 }
 
 pgFile *
-pgFileNew(const char *path, const char *rel_path, bool omit_symlink,
+pgFileNew(const char *path, const char *rel_path, bool follow_symlink,
 		  int external_dir_num, fio_location location)
 {
 	struct stat		st;
 	pgFile		   *file;
 
 	/* stat the file */
-	if (fio_stat(path, &st, omit_symlink, location) < 0)
+	if (fio_stat(path, &st, follow_symlink, location) < 0)
 	{
 		/* file not found is not an error case */
 		if (errno == ENOENT)
@@ -445,11 +445,11 @@ BlackListCompare(const void *str1, const void *str2)
  * List files, symbolic links and directories in the directory "root" and add
  * pgFile objects to "files".  We add "root" to "files" if add_root is true.
  *
- * When omit_symlink is true, symbolic link is ignored and only file or
+ * When follow_symlink is true, symbolic link is ignored and only file or
  * directory linked to will be listed.
  */
 void
-dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
+dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink,
 			  bool add_root, int external_dir_num, fio_location location)
 {
 	pgFile	   *file;
@@ -490,7 +490,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
 		parray_qsort(black_list, BlackListCompare);
 	}
 
-	file = pgFileNew(root, "", omit_symlink, external_dir_num, location);
+	file = pgFileNew(root, "", follow_symlink, external_dir_num, location);
 	if (file == NULL)
 	{
 		/* For external directory this is not ok */
@@ -512,7 +512,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
 	if (add_root)
 		parray_append(files, file);
 
-	dir_list_file_internal(files, file, exclude, omit_symlink, black_list,
+	dir_list_file_internal(files, file, exclude, follow_symlink, black_list,
 						   external_dir_num, location);
 
 	if (!add_root)
@@ -731,7 +731,7 @@ dir_check_file(pgFile *file)
  */
 static void
 dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
-					   bool omit_symlink, parray *black_list,
+					   bool follow_symlink, parray *black_list,
 					   int external_dir_num, fio_location location)
 {
 	DIR		    *dir;
@@ -764,7 +764,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
 		join_path_components(child, parent->path, dent->d_name);
 		join_path_components(rel_child, parent->rel_path, dent->d_name);
 
-		file = pgFileNew(child, rel_child, omit_symlink, external_dir_num,
+		file = pgFileNew(child, rel_child, follow_symlink, external_dir_num,
 						 location);
 		if (file == NULL)
 			continue;
@@ -821,7 +821,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
 		 * recursively.
 		 */
 		if (S_ISDIR(file->mode))
-			dir_list_file_internal(files, file, exclude, omit_symlink,
+			dir_list_file_internal(files, file, exclude, follow_symlink,
 								   black_list, external_dir_num, location);
 	}
 

src/dir.c

@@ -97,9 +97,11 @@ help_pg_probackup(void)
 	printf(_("                 [--remote-proto] [--remote-host]\n"));
 	printf(_("                 [--remote-port] [--remote-path] [--remote-user]\n"));
 	printf(_("                 [--ssh-options]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
 	printf(_("                 [--format=format]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME);
 	printf(_("                 [-D pgdata-path] [-C]\n"));
@@ -126,6 +128,7 @@ help_pg_probackup(void)
 	printf(_("                 [--remote-proto] [--remote-host]\n"));
 	printf(_("                 [--remote-port] [--remote-path] [--remote-user]\n"));
 	printf(_("                 [--ssh-options]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
 	printf(_("                 [-D pgdata-path] [-i backup-id] [-j num-threads]\n"));
@@ -143,6 +146,7 @@ help_pg_probackup(void)
 	printf(_("                 [--remote-proto] [--remote-host]\n"));
 	printf(_("                 [--remote-port] [--remote-path] [--remote-user]\n"));
 	printf(_("                 [--ssh-options]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME);
 	printf(_("                 [-i backup-id] [--progress] [-j num-threads]\n"));
@@ -151,32 +155,39 @@ help_pg_probackup(void)
 	printf(_("                 [--recovery-target-timeline=timeline]\n"));
 	printf(_("                 [--recovery-target-name=target-name]\n"));
 	printf(_("                 [--skip-block-validation]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME);
 	printf(_("                 [-D pgdata-path] [--progress] [-j num-threads]\n"));
 	printf(_("                 [--amcheck] [--skip-block-validation]\n"));
 	printf(_("                 [--heapallindexed]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s show -B backup-path\n"), PROGRAM_NAME);
 	printf(_("                 [--instance=instance_name [-i backup-id]]\n"));
 	printf(_("                 [--format=format]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
 	printf(_("                 [--wal] [-i backup-id | --expired | --merge-expired]\n"));
 	printf(_("                 [--dry-run]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
 	printf(_("                 -i backup-id [--progress] [-j num-threads]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME);
 	printf(_("                 --instance=instance_name\n"));
 	printf(_("                 [--external-dirs=external-directories-paths]\n"));
 	printf(_("                 [--remote-proto] [--remote-host]\n"));
 	printf(_("                 [--remote-port] [--remote-path] [--remote-user]\n"));
 	printf(_("                 [--ssh-options]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s del-instance -B backup-path\n"), PROGRAM_NAME);
 	printf(_("                 --instance=instance_name\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
 	printf(_("                 --wal-file-path=wal-file-path\n"));
@@ -188,13 +199,15 @@ help_pg_probackup(void)
 	printf(_("                 [--remote-proto] [--remote-host]\n"));
 	printf(_("                 [--remote-port] [--remote-path] [--remote-user]\n"));
 	printf(_("                 [--ssh-options]\n"));
+	printf(_("                 [--help]\n"));
 
 	printf(_("\n  %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
 	printf(_("                 --wal-file-path=wal-file-path\n"));
 	printf(_("                 --wal-file-name=wal-file-name\n"));
 	printf(_("                 [--remote-proto] [--remote-host]\n"));
 	printf(_("                 [--remote-port] [--remote-path] [--remote-user]\n"));
 	printf(_("                 [--ssh-options]\n"));
+	printf(_("                 [--help]\n"));
 
 	if ((PROGRAM_URL || PROGRAM_EMAIL))
 	{

src/help.c

@@ -577,6 +577,7 @@ extern bool in_backup_list(parray *backup_list, pgBackup *target_backup);
 extern int get_backup_index_number(parray *backup_list, pgBackup *backup);
 extern bool launch_agent(void);
 extern void launch_ssh(char* argv[]);
+extern void wait_ssh(void);
 
 #define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS
 #define COMPRESS_LEVEL_DEFAULT 1
@@ -586,7 +587,7 @@ extern const char* deparse_compress_alg(int alg);
 
 /* in dir.c */
 extern void dir_list_file(parray *files, const char *root, bool exclude,
-						  bool omit_symlink, bool add_root, int external_dir_num, fio_location location);
+						  bool follow_symlink, bool add_root, int external_dir_num, fio_location location);
 
 extern void create_data_directories(parray *dest_files,
 										const char *data_dir,
@@ -619,7 +620,7 @@ extern bool fileExists(const char *path, fio_location location);
 extern size_t pgFileSize(const char *path);
 
 extern pgFile *pgFileNew(const char *path, const char *rel_path,
-						 bool omit_symlink, int external_dir_num,
+						 bool follow_symlink, int external_dir_num,
 						 fio_location location);
 extern pgFile *pgFileInit(const char *path, const char *rel_path);
 extern void pgFileDelete(pgFile *file);