feat(storage): add support for 'skipIfExists' option for downloadMany (#2526)

* feature added

* fix

* Added system test and sample test cases

* copyright fix

* build: fix path-to-regexp to older version due to node 14 requirement

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Remove unnecessary sample code

* Manually modify README regards remove unnecessary sample code

* added 'skipIfExists' option for downloadMany

Node Transfer Manager: add support for 'skipIfExists' option for
downloadMany

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: harsha-accenture <[email protected]>

Author: 3 people

src/transfer-manager.ts

@@ -24,7 +24,7 @@ import {
 } from './file.js';
 import pLimit from 'p-limit';
 import * as path from 'path';
-import {createReadStream, promises as fsp} from 'fs';
+import {createReadStream, existsSync, promises as fsp} from 'fs';
 import {CRC32C} from './crc32c.js';
 import {GoogleAuth} from 'google-auth-library';
 import {XMLParser, XMLBuilder} from 'fast-xml-parser';
@@ -108,6 +108,7 @@ export interface DownloadManyFilesOptions {
   prefix?: string;
   stripPrefix?: string;
   passthroughOptions?: DownloadOptions;
+  skipIfExists?: boolean;
 }
 
 export interface DownloadFileInChunksOptions {
@@ -524,6 +525,8 @@ export class TransferManager {
    * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files.
    * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through
    * to each individual download operation.
+   * @property {boolean} [skipIfExists] Do not download the file if it already exists in
+   * the destination.
    *
    */
   /**
@@ -605,6 +608,12 @@ export class TransferManager {
       if (options.stripPrefix) {
         passThroughOptionsCopy.destination = file.name.replace(regex, '');
       }
+      if (
+        options.skipIfExists &&
+        existsSync(passThroughOptionsCopy.destination || '')
+      ) {
+        continue;
+      }
 
       promises.push(
         limit(async () => {

test/transfer-manager.ts

@@ -30,6 +30,7 @@ import {
   TransferManager,
   Storage,
   DownloadResponse,
+  DownloadManyFilesOptions,
 } from '../src/index.js';
 import assert from 'assert';
 import * as path from 'path';
@@ -195,6 +196,10 @@ describe('Transfer Manager', () => {
   });
 
   describe('downloadManyFiles', () => {
+    beforeEach(() => {
+      sandbox.stub(fs, 'existsSync').returns(true);
+    });
+
     it('calls download for each provided file', async () => {
       let count = 0;
       const firstFile = new File(bucket, 'first.txt');
@@ -276,6 +281,27 @@ describe('Transfer Manager', () => {
       await transferManager.downloadManyFiles([file], {passthroughOptions});
     });
 
+    it('does not download files that already exist locally when skipIfExists is true', async () => {
+      const firstFile = new File(bucket, 'first.txt');
+      sandbox.stub(firstFile, 'download').callsFake(options => {
+        assert.strictEqual(
+          (options as DownloadManyFilesOptions).skipIfExists,
+          0
+        );
+      });
+      const secondFile = new File(bucket, 'second.txt');
+      sandbox.stub(secondFile, 'download').callsFake(options => {
+        assert.strictEqual(
+          (options as DownloadManyFilesOptions).skipIfExists,
+          0
+        );
+      });
+
+      const files = [firstFile, secondFile];
+      const options = {skipIfExists: true};
+      await transferManager.downloadManyFiles(files, options);
+    });
+
     it('does not set the destination when prefix, strip prefix and passthroughOptions.destination are not provided', async () => {
       const options = {};
       const filename = 'first.txt';