[Core] Add NO_UPLOAD for remote_identity (#4307)

* Add skip flag to remote_identity

* Rename to NO_UPLOAD

* Fixes

* lint

* comments

* Add comments

* lint

Author: romilbhardwaj

docs/source/reference/config.rst

@@ -244,6 +244,10 @@ Available fields and semantics:
     # instances. SkyPilot will auto-create and reuse a service account (IAM
     # role) for AWS instances.
     #
+    # NO_UPLOAD: No credentials will be uploaded to the pods. Useful for
+    # avoiding overriding any existing credentials that may be automounted on
+    # the cluster.
+    #
     # Customized service account (IAM role): <string> or <list of single-element dict>
     # - <string>: apply the service account with the specified name to all instances.
     #    Example:
@@ -263,7 +267,8 @@ Available fields and semantics:
     #
     # - This only affects AWS instances. Local AWS credentials will still be
     #   uploaded to non-AWS instances (since those instances may need to access
-    #   AWS resources).
+    #   AWS resources). To fully disable credential upload, set
+    #   `remote_identity: NO_UPLOAD`.
     # - If the SkyPilot jobs/serve controller is on AWS, this setting will make
     #   non-AWS managed jobs / non-AWS service replicas fail to access any
     #   resources on AWS (since the controllers don't have AWS credential
@@ -406,11 +411,16 @@ Available fields and semantics:
     # instances. SkyPilot will auto-create and reuse a service account for GCP
     # instances.
     #
+    # NO_UPLOAD: No credentials will be uploaded to the pods. Useful for
+    # avoiding overriding any existing credentials that may be automounted on
+    # the cluster.
+    #
     # Two caveats of SERVICE_ACCOUNT for multicloud users:
     #
     # - This only affects GCP instances. Local GCP credentials will still be
     #   uploaded to non-GCP instances (since those instances may need to access
-    #   GCP resources).
+    #   GCP resources). To fully disable credential uploads, set
+    #   `remote_identity: NO_UPLOAD`.
     # - If the SkyPilot jobs/serve controller is on GCP, this setting will make
     #   non-GCP managed jobs / non-GCP service replicas fail to access any
     #   resources on GCP (since the controllers don't have GCP credential
@@ -497,6 +507,10 @@ Available fields and semantics:
     # SkyPilot will auto-create and reuse a service account with necessary roles
     # in the user's namespace.
     #
+    # NO_UPLOAD: No credentials will be uploaded to the pods. Useful for
+    # avoiding overriding any existing credentials that may be automounted on
+    # the cluster.
+    #
     # <string>: The name of a service account to use for all Kubernetes pods.
     # This service account must exist in the user's namespace and have all
     # necessary permissions. Refer to https://skypilot.readthedocs.io/en/latest/cloud-setup/cloud-permissions/kubernetes.html
@@ -505,7 +519,8 @@ Available fields and semantics:
     # Using SERVICE_ACCOUNT or a custom service account only affects Kubernetes
     # instances. Local ~/.kube/config will still be uploaded to non-Kubernetes
     # instances (e.g., a serve controller on GCP or AWS may need to provision
-    # Kubernetes resources).
+    # Kubernetes resources). To fully disable credential uploads, set
+    # `remote_identity: NO_UPLOAD`.
     #
     # Default: 'SERVICE_ACCOUNT'.
     remote_identity: my-k8s-service-account

sky/backends/backend_utils.py

@@ -690,26 +690,56 @@ def write_cluster_config(
         skypilot_config.get_nested(
             (str(to_provision.cloud).lower(), 'specific_reservations'), set()))
 
+    # Remote identity handling can have 4 cases:
+    # 1. LOCAL_CREDENTIALS (default for most clouds): Upload local credentials
+    # 2. SERVICE_ACCOUNT: SkyPilot creates and manages a service account
+    # 3. Custom service account: Use specified service account
+    # 4. NO_UPLOAD: Do not upload any credentials
+    #
+    # We need to upload credentials only if LOCAL_CREDENTIALS is specified. In
+    # other cases, we exclude the cloud from credential file uploads after
+    # running required checks.
     assert cluster_name is not None
-    excluded_clouds = []
+    excluded_clouds = set()
     remote_identity_config = skypilot_config.get_nested(
         (str(cloud).lower(), 'remote_identity'), None)
     remote_identity = schemas.get_default_remote_identity(str(cloud).lower())
     if isinstance(remote_identity_config, str):
         remote_identity = remote_identity_config
     if isinstance(remote_identity_config, list):
+        # Some clouds (e.g., AWS) support specifying multiple service accounts
+        # chosen based on the cluster name. Do the matching here to pick the
+        # correct one.
         for profile in remote_identity_config:
             if fnmatch.fnmatchcase(cluster_name, list(profile.keys())[0]):
                 remote_identity = list(profile.values())[0]
                 break
     if remote_identity != schemas.RemoteIdentityOptions.LOCAL_CREDENTIALS.value:
-        if not cloud.supports_service_account_on_remote():
+        # If LOCAL_CREDENTIALS is not specified, we add the cloud to the
+        # excluded_clouds set, but we must also check if the cloud supports
+        # service accounts.
+        if remote_identity == schemas.RemoteIdentityOptions.NO_UPLOAD.value:
+            # If NO_UPLOAD is specified, fall back to default remote identity
+            # for downstream logic but add it to excluded_clouds to skip
+            # credential file uploads.
+            remote_identity = schemas.get_default_remote_identity(
+                str(cloud).lower())
+        elif not cloud.supports_service_account_on_remote():
             raise exceptions.InvalidCloudConfigs(
                 'remote_identity: SERVICE_ACCOUNT is specified in '
                 f'{skypilot_config.loaded_config_path!r} for {cloud}, but it '
                 'is not supported by this cloud. Remove the config or set: '
                 '`remote_identity: LOCAL_CREDENTIALS`.')
-        excluded_clouds = [cloud]
+        excluded_clouds.add(cloud)
+
+    for cloud_str, cloud_obj in cloud_registry.CLOUD_REGISTRY.items():
+        remote_identity_config = skypilot_config.get_nested(
+            (cloud_str.lower(), 'remote_identity'), None)
+        if remote_identity_config:
+            if (remote_identity_config ==
+                    schemas.RemoteIdentityOptions.NO_UPLOAD.value):
+                excluded_clouds.add(cloud_obj)
+
     credentials = sky_check.get_cloud_credential_file_mounts(excluded_clouds)
 
     auth_config = {'ssh_private_key': auth.PRIVATE_SSH_KEY_PATH}

sky/utils/schemas.py

@@ -663,6 +663,7 @@ class RemoteIdentityOptions(enum.Enum):
     """
     LOCAL_CREDENTIALS = 'LOCAL_CREDENTIALS'
     SERVICE_ACCOUNT = 'SERVICE_ACCOUNT'
+    NO_UPLOAD = 'NO_UPLOAD'
 
 
 def get_default_remote_identity(cloud: str) -> str: