diff --git a/google/auth/compute_engine/_metadata.py b/google/auth/compute_engine/_metadata.py index fe821418e..94e4ffbf0 100644 --- a/google/auth/compute_engine/_metadata.py +++ b/google/auth/compute_engine/_metadata.py @@ -108,7 +108,9 @@ def ping(request, timeout=_METADATA_DEFAULT_TIMEOUT, retry_count=3): return False -def get(request, path, root=_METADATA_ROOT, recursive=False, retry_count=5): +def get( + request, path, root=_METADATA_ROOT, params=None, recursive=False, retry_count=5 +): """Fetch a resource from the metadata server. Args: @@ -117,6 +119,8 @@ def get(request, path, root=_METADATA_ROOT, recursive=False, retry_count=5): path (str): The resource to retrieve. For example, ``'instance/service-accounts/default'``. root (str): The full path to the metadata server root. + params (Optional[Mapping[str, str]]): A mapping of query parameter + keys to values. recursive (bool): Whether to do a recursive query of metadata. See https://cloud.google.com/compute/docs/metadata#aggcontents for more details. @@ -133,7 +137,7 @@ def get(request, path, root=_METADATA_ROOT, recursive=False, retry_count=5): retrieving metadata. """ base_url = urlparse.urljoin(root, path) - query_params = {} + query_params = {} if params is None else params if recursive: query_params["recursive"] = "true" @@ -224,11 +228,10 @@ def get_service_account_info(request, service_account="default"): google.auth.exceptions.TransportError: if an error occurred while retrieving metadata. """ - return get( - request, - "instance/service-accounts/{0}/".format(service_account), - recursive=True, - ) + path = "instance/service-accounts/{0}/".format(service_account) + # See https://cloud.google.com/compute/docs/metadata#aggcontents + # for more on the use of 'recursive'. + return get(request, path, params={"recursive": "true"}) def get_service_account_token(request, service_account="default"): diff --git a/google/auth/compute_engine/credentials.py b/google/auth/compute_engine/credentials.py index b7fca1832..8a41ffcc0 100644 --- a/google/auth/compute_engine/credentials.py +++ b/google/auth/compute_engine/credentials.py @@ -323,12 +323,9 @@ def _call_metadata_identity_endpoint(self, request): ValueError: If extracting expiry from the obtained ID token fails. """ try: - id_token = _metadata.get( - request, - "instance/service-accounts/default/identity?audience={}&format=full".format( - self._target_audience - ), - ) + path = "instance/service-accounts/default/identity" + params = {"audience": self._target_audience, "format": "full"} + id_token = _metadata.get(request, path, params=params) except exceptions.TransportError as caught_exc: new_exc = exceptions.RefreshError(caught_exc) six.raise_from(new_exc, caught_exc) diff --git a/google/auth/external_account.py b/google/auth/external_account.py index 0a6c3cab2..ab91fd8d7 100644 --- a/google/auth/external_account.py +++ b/google/auth/external_account.py @@ -28,6 +28,7 @@ import abc import datetime +import json import six @@ -42,6 +43,8 @@ _STS_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" # The token exchange requested_token_type. This is always an access_token. _STS_REQUESTED_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token" +# Cloud resource manager URL used to retrieve project information. +_CLOUD_RESOURCE_MANAGER = "https://cloudresourcemanager.googleapis.com/v1/projects/" @six.add_metaclass(abc.ABCMeta) @@ -107,6 +110,7 @@ def __init__( self._impersonated_credentials = self._initialize_impersonated_credentials() else: self._impersonated_credentials = None + self._project_id = None @property def requires_scopes(self): @@ -117,6 +121,20 @@ def requires_scopes(self): """ return True if not self._scopes else False + @property + def project_number(self): + """Optional[str]: The project number corresponding to the workload identity pool.""" + + # STS audience pattern: + # //iam.googleapis.com/projects/$PROJECT_NUMBER/locations/... + components = self._audience.split("/") + try: + project_index = components.index("projects") + if project_index + 1 < len(components): + return components[project_index + 1] or None + except ValueError: + return None + @_helpers.copy_docstring(credentials.Scoped) def with_scopes(self, scopes): return self.__class__( @@ -144,6 +162,49 @@ def retrieve_subject_token(self, request): # (pylint doesn't recognize that this is abstract) raise NotImplementedError("retrieve_subject_token must be implemented") + def get_project_id(self, request): + """Retrieves the project ID corresponding to the workload identity pool. + + When not determinable, None is returned. + + This is introduced to support the current pattern of using the Auth library: + + credentials, project_id = google.auth.default() + + The resource may not have permission (resourcemanager.projects.get) to + call this API or the required scopes may not be selected: + https://cloud.google.com/resource-manager/reference/rest/v1/projects/get#authorization-scopes + + Args: + request (google.auth.transport.Request): A callable used to make + HTTP requests. + Returns: + Optional[str]: The project ID corresponding to the workload identity pool + if determinable. + """ + if self._project_id: + # If already retrieved, return the cached project ID value. + return self._project_id + if self.project_number: + headers = {} + url = _CLOUD_RESOURCE_MANAGER + self.project_number + self.before_request(request, "GET", url, headers) + response = request(url=url, method="GET", headers=headers) + + response_body = ( + response.data.decode("utf-8") + if hasattr(response.data, "decode") + else response.data + ) + response_data = json.loads(response_body) + + if response.status == 200: + # Cache result as this field is immutable. + self._project_id = response_data.get("projectId") + return self._project_id + + return None + @_helpers.copy_docstring(credentials.Credentials) def refresh(self, request): if self._impersonated_credentials: diff --git a/noxfile.py b/noxfile.py index f4b909cff..79a09e4da 100644 --- a/noxfile.py +++ b/noxfile.py @@ -142,6 +142,7 @@ def docs(session): @nox.session(python="pypy") def pypy(session): session.install(*TEST_DEPENDENCIES) + session.install(*ASYNC_DEPENDENCIES) session.install(".") session.run( "pytest", diff --git a/tests/compute_engine/test__metadata.py b/tests/compute_engine/test__metadata.py index d9b039a32..d05337263 100644 --- a/tests/compute_engine/test__metadata.py +++ b/tests/compute_engine/test__metadata.py @@ -155,6 +155,49 @@ def test_get_success_text(): assert result == data +def test_get_success_params(): + data = "foobar" + request = make_request(data, headers={"content-type": "text/plain"}) + params = {"recursive": "true"} + + result = _metadata.get(request, PATH, params=params) + + request.assert_called_once_with( + method="GET", + url=_metadata._METADATA_ROOT + PATH + "?recursive=true", + headers=_metadata._METADATA_HEADERS, + ) + assert result == data + + +def test_get_success_recursive_and_params(): + data = "foobar" + request = make_request(data, headers={"content-type": "text/plain"}) + params = {"recursive": "false"} + result = _metadata.get(request, PATH, recursive=True, params=params) + + request.assert_called_once_with( + method="GET", + url=_metadata._METADATA_ROOT + PATH + "?recursive=true", + headers=_metadata._METADATA_HEADERS, + ) + assert result == data + + +def test_get_success_recursive(): + data = "foobar" + request = make_request(data, headers={"content-type": "text/plain"}) + + result = _metadata.get(request, PATH, recursive=True) + + request.assert_called_once_with( + method="GET", + url=_metadata._METADATA_ROOT + PATH + "?recursive=true", + headers=_metadata._METADATA_HEADERS, + ) + assert result == data + + def test_get_success_custom_root_new_variable(): request = make_request("{}", headers={"content-type": "application/json"}) diff --git a/tests/test_external_account.py b/tests/test_external_account.py index c33933979..89b64bdce 100644 --- a/tests/test_external_account.py +++ b/tests/test_external_account.py @@ -71,7 +71,7 @@ class TestCredentials(object): POOL_ID = "POOL_ID" PROVIDER_ID = "PROVIDER_ID" AUDIENCE = ( - "//iam.googleapis.com/project/{}" + "//iam.googleapis.com/projects/{}" "/locations/global/workloadIdentityPools/{}" "/providers/{}" ).format(PROJECT_NUMBER, POOL_ID, PROVIDER_ID) @@ -102,6 +102,18 @@ class TestCredentials(object): "status": "INVALID_ARGUMENT", } } + PROJECT_ID = "my-proj-id" + CLOUD_RESOURCE_MANAGER_URL = ( + "https://cloudresourcemanager.googleapis.com/v1/projects/" + ) + CLOUD_RESOURCE_MANAGER_SUCCESS_RESPONSE = { + "projectNumber": PROJECT_NUMBER, + "projectId": PROJECT_ID, + "lifecycleState": "ACTIVE", + "name": "project-name", + "createTime": "2018-11-06T04:42:54.109Z", + "parent": {"type": "folder", "id": "12345678901"}, + } @classmethod def make_credentials( @@ -127,10 +139,12 @@ def make_credentials( @classmethod def make_mock_request( cls, - data, status=http_client.OK, - impersonation_data=None, + data=None, impersonation_status=None, + impersonation_data=None, + cloud_resource_manager_status=None, + cloud_resource_manager_data=None, ): # STS token exchange request. token_response = mock.create_autospec(transport.Response, instance=True) @@ -139,7 +153,7 @@ def make_mock_request( responses = [token_response] # If service account impersonation is requested, mock the expected response. - if impersonation_status and impersonation_status: + if impersonation_status: impersonation_response = mock.create_autospec( transport.Response, instance=True ) @@ -147,6 +161,17 @@ def make_mock_request( impersonation_response.data = json.dumps(impersonation_data).encode("utf-8") responses.append(impersonation_response) + # If cloud resource manager is requested, mock the expected response. + if cloud_resource_manager_status: + cloud_resource_manager_response = mock.create_autospec( + transport.Response, instance=True + ) + cloud_resource_manager_response.status = cloud_resource_manager_status + cloud_resource_manager_response.data = json.dumps( + cloud_resource_manager_data + ).encode("utf-8") + responses.append(cloud_resource_manager_response) + request = mock.create_autospec(transport.Request) request.side_effect = responses @@ -172,6 +197,15 @@ def assert_impersonation_request_kwargs(cls, request_kwargs, headers, request_da body_json = json.loads(request_kwargs["body"].decode("utf-8")) assert body_json == request_data + @classmethod + def assert_resource_manager_request_kwargs( + cls, request_kwargs, project_number, headers + ): + assert request_kwargs["url"] == cls.CLOUD_RESOURCE_MANAGER_URL + project_number + assert request_kwargs["method"] == "GET" + assert request_kwargs["headers"] == headers + assert "body" not in request_kwargs + def test_default_state(self): credentials = self.make_credentials() @@ -709,3 +743,142 @@ def test_before_request_impersonation_expired(self, utcnow): assert headers == { "authorization": "Bearer {}".format(impersonation_response["accessToken"]) } + + @pytest.mark.parametrize( + "audience", + [ + # Legacy K8s audience format. + "identitynamespace:1f12345:my_provider", + # Unrealistic audiences. + "//iam.googleapis.com/projects", + "//iam.googleapis.com/projects/", + "//iam.googleapis.com/project/123456", + "//iam.googleapis.com/projects//123456", + "//iam.googleapis.com/prefix_projects/123456", + "//iam.googleapis.com/projects_suffix/123456", + ], + ) + def test_project_number_indeterminable(self, audience): + credentials = CredentialsImpl( + audience=audience, + subject_token_type=self.SUBJECT_TOKEN_TYPE, + token_url=self.TOKEN_URL, + credential_source=self.CREDENTIAL_SOURCE, + ) + + assert credentials.project_number is None + assert credentials.get_project_id(None) is None + + def test_project_number_determinable(self): + credentials = CredentialsImpl( + audience=self.AUDIENCE, + subject_token_type=self.SUBJECT_TOKEN_TYPE, + token_url=self.TOKEN_URL, + credential_source=self.CREDENTIAL_SOURCE, + ) + + assert credentials.project_number == self.PROJECT_NUMBER + + def test_get_project_id_cloud_resource_manager_success(self): + # STS token exchange request/response. + token_response = self.SUCCESS_RESPONSE.copy() + token_headers = {"Content-Type": "application/x-www-form-urlencoded"} + token_request_data = { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "audience": self.AUDIENCE, + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + "subject_token": "subject_token_0", + "subject_token_type": self.SUBJECT_TOKEN_TYPE, + "scope": "https://www.googleapis.com/auth/iam", + } + # Service account impersonation request/response. + expire_time = ( + _helpers.utcnow().replace(microsecond=0) + datetime.timedelta(seconds=3600) + ).isoformat("T") + "Z" + expected_expiry = datetime.datetime.strptime(expire_time, "%Y-%m-%dT%H:%M:%SZ") + impersonation_response = { + "accessToken": "SA_ACCESS_TOKEN", + "expireTime": expire_time, + } + impersonation_headers = { + "Content-Type": "application/json", + "x-goog-user-project": self.QUOTA_PROJECT_ID, + "authorization": "Bearer {}".format(token_response["access_token"]), + } + impersonation_request_data = { + "delegates": None, + "scope": self.SCOPES, + "lifetime": "3600s", + } + # Initialize mock request to handle token exchange, service account + # impersonation and cloud resource manager request. + request = self.make_mock_request( + status=http_client.OK, + data=self.SUCCESS_RESPONSE.copy(), + impersonation_status=http_client.OK, + impersonation_data=impersonation_response, + cloud_resource_manager_status=http_client.OK, + cloud_resource_manager_data=self.CLOUD_RESOURCE_MANAGER_SUCCESS_RESPONSE, + ) + credentials = self.make_credentials( + service_account_impersonation_url=self.SERVICE_ACCOUNT_IMPERSONATION_URL, + scopes=self.SCOPES, + quota_project_id=self.QUOTA_PROJECT_ID, + ) + + # Expected project ID from cloud resource manager response should be returned. + project_id = credentials.get_project_id(request) + + assert project_id == self.PROJECT_ID + # 3 requests should be processed. + assert len(request.call_args_list) == 3 + # Verify token exchange request parameters. + self.assert_token_request_kwargs( + request.call_args_list[0].kwargs, token_headers, token_request_data + ) + # Verify service account impersonation request parameters. + self.assert_impersonation_request_kwargs( + request.call_args_list[1].kwargs, + impersonation_headers, + impersonation_request_data, + ) + # In the process of getting project ID, an access token should be + # retrieved. + assert credentials.valid + assert credentials.expiry == expected_expiry + assert not credentials.expired + assert credentials.token == impersonation_response["accessToken"] + # Verify cloud resource manager request parameters. + self.assert_resource_manager_request_kwargs( + request.call_args_list[2].kwargs, + self.PROJECT_NUMBER, + { + "x-goog-user-project": self.QUOTA_PROJECT_ID, + "authorization": "Bearer {}".format( + impersonation_response["accessToken"] + ), + }, + ) + + # Calling get_project_id again should return the cached project_id. + project_id = credentials.get_project_id(request) + + assert project_id == self.PROJECT_ID + # No additional requests. + assert len(request.call_args_list) == 3 + + def test_get_project_id_cloud_resource_manager_error(self): + # Simulate resource doesn't have sufficient permissions to access + # cloud resource manager. + request = self.make_mock_request( + status=http_client.OK, + data=self.SUCCESS_RESPONSE.copy(), + cloud_resource_manager_status=http_client.UNAUTHORIZED, + ) + credentials = self.make_credentials() + + project_id = credentials.get_project_id(request) + + assert project_id is None + # Only 2 requests to STS and cloud resource manager should be sent. + assert len(request.call_args_list) == 2