feat: support custom universe domains/TPC (#1212)

Author: andrewsg

google/cloud/storage/_helpers.py

@@ -21,6 +21,7 @@
 from hashlib import md5
 import os
 from urllib.parse import urlsplit
+from urllib.parse import urlunsplit
 from uuid import uuid4
 
 from google import resumable_media
@@ -30,19 +31,24 @@
 from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
 
 
-STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
+STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"  # Despite name, includes scheme.
 """Environment variable defining host for Storage emulator."""
 
-_API_ENDPOINT_OVERRIDE_ENV_VAR = "API_ENDPOINT_OVERRIDE"
+_API_ENDPOINT_OVERRIDE_ENV_VAR = "API_ENDPOINT_OVERRIDE"  # Includes scheme.
 """This is an experimental configuration variable. Use api_endpoint instead."""
 
 _API_VERSION_OVERRIDE_ENV_VAR = "API_VERSION_OVERRIDE"
 """This is an experimental configuration variable used for internal testing."""
 
-_DEFAULT_STORAGE_HOST = os.getenv(
-    _API_ENDPOINT_OVERRIDE_ENV_VAR, "https://storage.googleapis.com"
+_DEFAULT_UNIVERSE_DOMAIN = "googleapis.com"
+
+_STORAGE_HOST_TEMPLATE = "storage.{universe_domain}"
+
+_TRUE_DEFAULT_STORAGE_HOST = _STORAGE_HOST_TEMPLATE.format(
+    universe_domain=_DEFAULT_UNIVERSE_DOMAIN
 )
-"""Default storage host for JSON API."""
+
+_DEFAULT_SCHEME = "https://"
 
 _API_VERSION = os.getenv(_API_VERSION_OVERRIDE_ENV_VAR, "v1")
 """API version of the default storage host"""
@@ -72,8 +78,39 @@
 )
 
 
-def _get_storage_host():
-    return os.environ.get(STORAGE_EMULATOR_ENV_VAR, _DEFAULT_STORAGE_HOST)
+def _get_storage_emulator_override():
+    return os.environ.get(STORAGE_EMULATOR_ENV_VAR, None)
+
+
+def _get_default_storage_base_url():
+    return os.getenv(
+        _API_ENDPOINT_OVERRIDE_ENV_VAR, _DEFAULT_SCHEME + _TRUE_DEFAULT_STORAGE_HOST
+    )
+
+
+def _get_api_endpoint_override():
+    """This is an experimental configuration variable. Use api_endpoint instead."""
+    if _get_default_storage_base_url() != _DEFAULT_SCHEME + _TRUE_DEFAULT_STORAGE_HOST:
+        return _get_default_storage_base_url()
+    return None
+
+
+def _virtual_hosted_style_base_url(url, bucket, trailing_slash=False):
+    """Returns the scheme and netloc sections of the url, with the bucket
+    prepended to the netloc.
+
+    Not intended for use with netlocs which include a username and password.
+    """
+    parsed_url = urlsplit(url)
+    new_netloc = f"{bucket}.{parsed_url.netloc}"
+    base_url = urlunsplit(
+        (parsed_url.scheme, new_netloc, "/" if trailing_slash else "", "", "")
+    )
+    return base_url
+
+
+def _use_client_cert():
+    return os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true"
 
 
 def _get_environ_project():

google/cloud/storage/_http.py

@@ -21,8 +21,14 @@
 
 
 class Connection(_http.JSONConnection):
-    """A connection to Google Cloud Storage via the JSON REST API. Mutual TLS feature will be
-    enabled if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "true".
+    """A connection to Google Cloud Storage via the JSON REST API.
+
+    Mutual TLS will be enabled if the "GOOGLE_API_USE_CLIENT_CERTIFICATE"
+    environment variable is set to the exact string "true" (case-sensitive).
+
+    Mutual TLS is not compatible with any API endpoint or universe domain
+    override at this time. If such settings are enabled along with
+    "GOOGLE_API_USE_CLIENT_CERTIFICATE", a ValueError will be raised.
 
     :type client: :class:`~google.cloud.storage.client.Client`
     :param client: The client that owns the current connection.
@@ -34,7 +40,7 @@ class Connection(_http.JSONConnection):
     :param api_endpoint: (Optional) api endpoint to use.
     """
 
-    DEFAULT_API_ENDPOINT = _helpers._DEFAULT_STORAGE_HOST
+    DEFAULT_API_ENDPOINT = _helpers._get_default_storage_base_url()
     DEFAULT_API_MTLS_ENDPOINT = "https://storage.mtls.googleapis.com"
 
     def __init__(self, client, client_info=None, api_endpoint=None):

google/cloud/storage/_signing.py

@@ -466,7 +466,7 @@ def generate_signed_url_v4(
                        ``tzinfo`` set,  it will be assumed to be ``UTC``.
 
     :type api_access_endpoint: str
-    :param api_access_endpoint: (Optional) URI base. Defaults to
+    :param api_access_endpoint: URI base. Defaults to
                                 "https://storage.googleapis.com/"
 
     :type method: str

google/cloud/storage/blob.py

@@ -57,11 +57,12 @@
 from google.cloud.storage._helpers import _raise_if_more_than_one_set
 from google.cloud.storage._helpers import _api_core_retry_to_resumable_media_retry
 from google.cloud.storage._helpers import _get_default_headers
+from google.cloud.storage._helpers import _get_default_storage_base_url
 from google.cloud.storage._signing import generate_signed_url_v2
 from google.cloud.storage._signing import generate_signed_url_v4
 from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE
-from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
 from google.cloud.storage._helpers import _API_VERSION
+from google.cloud.storage._helpers import _virtual_hosted_style_base_url
 from google.cloud.storage.acl import ACL
 from google.cloud.storage.acl import ObjectACL
 from google.cloud.storage.constants import _DEFAULT_TIMEOUT
@@ -80,7 +81,6 @@
 from google.cloud.storage.fileio import BlobWriter
 
 
-_API_ACCESS_ENDPOINT = _DEFAULT_STORAGE_HOST
 _DEFAULT_CONTENT_TYPE = "application/octet-stream"
 _DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/{api_version}{path}?alt=media"
 _BASE_UPLOAD_TEMPLATE = (
@@ -376,8 +376,12 @@ def public_url(self):
         :rtype: `string`
         :returns: The public URL for this blob.
         """
+        if self.client:
+            endpoint = self.client.api_endpoint
+        else:
+            endpoint = _get_default_storage_base_url()
         return "{storage_base_url}/{bucket_name}/{quoted_name}".format(
-            storage_base_url=_API_ACCESS_ENDPOINT,
+            storage_base_url=endpoint,
             bucket_name=self.bucket.name,
             quoted_name=_quote(self.name, safe=b"/~"),
         )
@@ -416,7 +420,7 @@ def from_string(cls, uri, client=None):
     def generate_signed_url(
         self,
         expiration=None,
-        api_access_endpoint=_API_ACCESS_ENDPOINT,
+        api_access_endpoint=None,
         method="GET",
         content_md5=None,
         content_type=None,
@@ -464,7 +468,9 @@ def generate_signed_url(
             assumed to be ``UTC``.
 
         :type api_access_endpoint: str
-        :param api_access_endpoint: (Optional) URI base.
+        :param api_access_endpoint: (Optional) URI base, for instance
+            "https://storage.googleapis.com". If not specified, the client's
+            api_endpoint will be used. Incompatible with bucket_bound_hostname.
 
         :type method: str
         :param method: The HTTP verb that will be used when requesting the URL.
@@ -537,21 +543,22 @@ def generate_signed_url(
         :param virtual_hosted_style:
             (Optional) If true, then construct the URL relative the bucket's
             virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
+            Incompatible with bucket_bound_hostname.
 
         :type bucket_bound_hostname: str
         :param bucket_bound_hostname:
-            (Optional) If passed, then construct the URL relative to the
-            bucket-bound hostname.  Value can be a bare or with scheme, e.g.,
-            'example.com' or 'http://example.com'.  See:
-            https://cloud.google.com/storage/docs/request-endpoints#cname
+            (Optional) If passed, then construct the URL relative to the bucket-bound hostname.
+            Value can be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
+            Incompatible with api_access_endpoint and virtual_hosted_style.
+            See: https://cloud.google.com/storage/docs/request-endpoints#cname
 
         :type scheme: str
         :param scheme:
             (Optional) If ``bucket_bound_hostname`` is passed as a bare
             hostname, use this value as the scheme.  ``https`` will work only
             when using a CDN.  Defaults to ``"http"``.
 
-        :raises: :exc:`ValueError` when version is invalid.
+        :raises: :exc:`ValueError` when version is invalid or mutually exclusive arguments are used.
         :raises: :exc:`TypeError` when expiration is not a valid type.
         :raises: :exc:`AttributeError` if credentials is not an instance
                 of :class:`google.auth.credentials.Signing`.
@@ -565,25 +572,38 @@ def generate_signed_url(
         elif version not in ("v2", "v4"):
             raise ValueError("'version' must be either 'v2' or 'v4'")
 
+        if (
+            api_access_endpoint is not None or virtual_hosted_style
+        ) and bucket_bound_hostname:
+            raise ValueError(
+                "The bucket_bound_hostname argument is not compatible with "
+                "either api_access_endpoint or virtual_hosted_style."
+            )
+
+        if api_access_endpoint is None:
+            client = self._require_client(client)
+            api_access_endpoint = client.api_endpoint
+
         quoted_name = _quote(self.name, safe=b"/~")
 
         # If you are on Google Compute Engine, you can't generate a signed URL
         # using GCE service account.
         # See https://github.com/googleapis/google-auth-library-python/issues/50
         if virtual_hosted_style:
-            api_access_endpoint = f"https://{self.bucket.name}.storage.googleapis.com"
+            api_access_endpoint = _virtual_hosted_style_base_url(
+                api_access_endpoint, self.bucket.name
+            )
+            resource = f"/{quoted_name}"
         elif bucket_bound_hostname:
             api_access_endpoint = _bucket_bound_hostname_url(
                 bucket_bound_hostname, scheme
             )
+            resource = f"/{quoted_name}"
         else:
             resource = f"/{self.bucket.name}/{quoted_name}"
 
-        if virtual_hosted_style or bucket_bound_hostname:
-            resource = f"/{quoted_name}"
-
         if credentials is None:
-            client = self._require_client(client)
+            client = self._require_client(client)  # May be redundant, but that's ok.
             credentials = client._credentials
 
         if version == "v2":

google/cloud/storage/bucket.py

@@ -36,6 +36,7 @@
 from google.cloud.storage._signing import generate_signed_url_v2
 from google.cloud.storage._signing import generate_signed_url_v4
 from google.cloud.storage._helpers import _bucket_bound_hostname_url
+from google.cloud.storage._helpers import _virtual_hosted_style_base_url
 from google.cloud.storage.acl import BucketACL
 from google.cloud.storage.acl import DefaultObjectACL
 from google.cloud.storage.blob import Blob
@@ -82,7 +83,6 @@
     "valid before the bucket is created. Instead, pass the location "
     "to `Bucket.create`."
 )
-_API_ACCESS_ENDPOINT = "https://storage.googleapis.com"
 
 
 def _blobs_page_start(iterator, page, response):
@@ -3265,7 +3265,7 @@ def lock_retention_policy(
     def generate_signed_url(
         self,
         expiration=None,
-        api_access_endpoint=_API_ACCESS_ENDPOINT,
+        api_access_endpoint=None,
         method="GET",
         headers=None,
         query_parameters=None,
@@ -3298,7 +3298,9 @@ def generate_signed_url(
                            ``tzinfo`` set,  it will be assumed to be ``UTC``.
 
         :type api_access_endpoint: str
-        :param api_access_endpoint: (Optional) URI base.
+        :param api_access_endpoint: (Optional) URI base, for instance
+            "https://storage.googleapis.com". If not specified, the client's
+            api_endpoint will be used. Incompatible with bucket_bound_hostname.
 
         :type method: str
         :param method: The HTTP verb that will be used when requesting the URL.
@@ -3322,7 +3324,6 @@ def generate_signed_url(
         :param client: (Optional) The client to use.  If not passed, falls back
                        to the ``client`` stored on the blob's bucket.
 
-
         :type credentials: :class:`google.auth.credentials.Credentials` or
                            :class:`NoneType`
         :param credentials: The authorization credentials to attach to requests.
@@ -3338,11 +3339,13 @@ def generate_signed_url(
         :param virtual_hosted_style:
             (Optional) If true, then construct the URL relative the bucket's
             virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
+            Incompatible with bucket_bound_hostname.
 
         :type bucket_bound_hostname: str
         :param bucket_bound_hostname:
-            (Optional) If pass, then construct the URL relative to the bucket-bound hostname.
-            Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
+            (Optional) If passed, then construct the URL relative to the bucket-bound hostname.
+            Value can be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
+            Incompatible with api_access_endpoint and virtual_hosted_style.
             See: https://cloud.google.com/storage/docs/request-endpoints#cname
 
         :type scheme: str
@@ -3351,7 +3354,7 @@ def generate_signed_url(
             this value as the scheme.  ``https`` will work only when using a CDN.
             Defaults to ``"http"``.
 
-        :raises: :exc:`ValueError` when version is invalid.
+        :raises: :exc:`ValueError` when version is invalid or mutually exclusive arguments are used.
         :raises: :exc:`TypeError` when expiration is not a valid type.
         :raises: :exc:`AttributeError` if credentials is not an instance
                 of :class:`google.auth.credentials.Signing`.
@@ -3365,23 +3368,36 @@ def generate_signed_url(
         elif version not in ("v2", "v4"):
             raise ValueError("'version' must be either 'v2' or 'v4'")
 
+        if (
+            api_access_endpoint is not None or virtual_hosted_style
+        ) and bucket_bound_hostname:
+            raise ValueError(
+                "The bucket_bound_hostname argument is not compatible with "
+                "either api_access_endpoint or virtual_hosted_style."
+            )
+
+        if api_access_endpoint is None:
+            client = self._require_client(client)
+            api_access_endpoint = client.api_endpoint
+
         # If you are on Google Compute Engine, you can't generate a signed URL
         # using GCE service account.
         # See https://github.com/googleapis/google-auth-library-python/issues/50
         if virtual_hosted_style:
-            api_access_endpoint = f"https://{self.name}.storage.googleapis.com"
+            api_access_endpoint = _virtual_hosted_style_base_url(
+                api_access_endpoint, self.name
+            )
+            resource = "/"
         elif bucket_bound_hostname:
             api_access_endpoint = _bucket_bound_hostname_url(
                 bucket_bound_hostname, scheme
             )
+            resource = "/"
         else:
             resource = f"/{self.name}"
 
-        if virtual_hosted_style or bucket_bound_hostname:
-            resource = "/"
-
         if credentials is None:
-            client = self._require_client(client)
+            client = self._require_client(client)  # May be redundant, but that's ok.
             credentials = client._credentials
 
         if version == "v2":