revise code

Author: DanielZhangQD

sky/adaptors/gcp.py

@@ -68,6 +68,13 @@ def credential_error_exception():
     return exceptions.DefaultCredentialsError
 
 
+@common.load_lazy_modules(_LAZY_MODULES)
+def auth_error_exception():
+    """GoogleAuthError exception."""
+    from google.auth import exceptions
+    return exceptions.GoogleAuthError
+
+
 @common.load_lazy_modules(_LAZY_MODULES)
 def gcp_auth_refresh_error_exception():
     """GCP auth refresh error exception."""

sky/authentication.py

@@ -36,6 +36,7 @@
 import yaml
 
 from sky import clouds
+from sky import exceptions
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import gcp
@@ -204,6 +205,11 @@ def setup_gcp_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
             sys.exit(1)
         else:
             raise
+    except gcp.auth_error_exception() as e:
+        logger.error(
+            f'Error getting GCP project: {common_utils.format_exception(e)}')
+        raise exceptions.InvalidCloudCredentials(
+            f'{common_utils.format_exception(e)}')
     except socket.timeout:
         logger.error('Socket timed out when trying to get the GCP project. '
                      'Please check your network connection.')

sky/backends/backend_utils.py

@@ -806,7 +806,14 @@ def write_cluster_config(
             logger.warning(f'Failed to calculate config_hash: {e}')
             logger.debug('Full exception:', exc_info=e)
         return config_dict
-    _add_auth_to_cluster_config(cloud, tmp_yaml_path)
+
+    try:
+        _add_auth_to_cluster_config(cloud, tmp_yaml_path)
+    except exceptions.InvalidCloudCredentials as e:
+        raise e
+    except Exception as e:  # pylint: disable=broad-except
+        logger.warning(f'Failed to add auth to cluster config: {e}')
+        logger.debug('Full exception:', exc_info=e)
 
     # Restore the old yaml content for backward compatibility.
     if os.path.exists(yaml_path) and keep_launch_fields_in_existing_config:

sky/backends/cloud_vm_ray_backend.py

@@ -1098,7 +1098,7 @@ def _aws_handler(blocked_resources: Set['resources_lib.Resources'],
         output = str(error)
         logger.info(f'AWS handler error: {output}')
         # Block AWS if the credential has expired.
-        if output.find('CloudCredentialExpired') != -1:
+        if output.find('InvalidCloudCredentials') != -1:
             _add_to_blocked_resources(
                 blocked_resources, resources_lib.Resources(cloud=clouds.AWS()))
         else:
@@ -1435,6 +1435,17 @@ def _retry_zones(
                 # does not have nodes labeled with GPU types.
                 logger.info(f'{e}')
                 continue
+            except exceptions.InvalidCloudCredentials as e:
+                # Failed due to invalid cloud credentials.
+                logger.warning(f'{common_utils.format_exception(e)}')
+                # We should block the entire cloud for invalid cloud credentials
+                _add_to_blocked_resources(
+                    self._blocked_resources,
+                    to_provision.copy(region=None, zone=None))
+                raise exceptions.ResourcesUnavailableError(
+                    f'Failed to provision on cloud {to_provision.cloud} due to '
+                    f'invalid cloud credentials: '
+                    f'{common_utils.format_exception(e)}')
             except exceptions.InvalidCloudConfigs as e:
                 # Failed due to invalid user configs in ~/.sky/config.yaml.
                 logger.warning(f'{common_utils.format_exception(e)}')

sky/exceptions.py

@@ -180,8 +180,8 @@ class InvalidCloudConfigs(Exception):
     pass
 
 
-class CloudCredentialExpired(Exception):
-    """Raised when the cloud credentials have expired."""
+class InvalidCloudCredentials(Exception):
+    """Raised when the cloud credentials are invalid."""
     pass
 
 

sky/provision/aws/utils.py

@@ -74,10 +74,10 @@ def handle_boto_error(exc: Exception, msg: str) -> None:
             f'{colorama.Style.RESET_ALL}\n'
             f'You can find a script that automates this at:'
             f'{aws_session_script_url}')
-        # Raise the CloudCredentialExpired exception so that
+        # Raise the InvalidCloudCredentials exception so that
         # the provisioner can failover to other clouds
-        raise exceptions.CloudCredentialExpired(
-            f'CloudCredentialExpired: {generic_message}') from exc
+        raise exceptions.InvalidCloudCredentials(
+            f'InvalidCloudCredentials: {generic_message}') from exc
 
     # todo: any other errors that we should catch separately?
 

sky/provision/provisioner.py

@@ -145,7 +145,7 @@ def bulk_provision(
         except exceptions.NoClusterLaunchedError:
             # Skip the teardown if the cluster was never launched.
             raise
-        except exceptions.CloudCredentialExpired:
+        except exceptions.InvalidCloudCredentials:
             # Skip the teardown if the cloud config is expired and
             # the provisioner should failover to other clouds.
             raise