diff --git a/google/auth/_default.py b/google/auth/_default.py index de81c5b2c..3d8ce04ff 100644 --- a/google/auth/_default.py +++ b/google/auth/_default.py @@ -34,7 +34,8 @@ # Valid types accepted for file-based credentials. _AUTHORIZED_USER_TYPE = "authorized_user" _SERVICE_ACCOUNT_TYPE = "service_account" -_VALID_TYPES = (_AUTHORIZED_USER_TYPE, _SERVICE_ACCOUNT_TYPE) +_EXTERNAL_ACCOUNT_TYPE = "external_account" +_VALID_TYPES = (_AUTHORIZED_USER_TYPE, _SERVICE_ACCOUNT_TYPE, _EXTERNAL_ACCOUNT_TYPE) # Help message when no credentials can be found. _HELP_MESSAGE = """\ @@ -69,11 +70,13 @@ def _warn_about_problematic_credentials(credentials): warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING) -def load_credentials_from_file(filename, scopes=None, quota_project_id=None): +def load_credentials_from_file( + filename, scopes=None, quota_project_id=None, request=None +): """Loads Google credentials from a file. - The credentials file must be a service account key or stored authorized - user credentials. + The credentials file must be a service account key, stored authorized + user credentials or external account credentials. Args: filename (str): The full path to the credentials file. @@ -81,12 +84,18 @@ def load_credentials_from_file(filename, scopes=None, quota_project_id=None): specified, the credentials will automatically be scoped if necessary quota_project_id (Optional[str]): The project ID used for - quota and billing. + quota and billing. + request (Optional[google.auth.transport.Request]): An object used to make + HTTP requests. This is used to determine the associated project ID + for a workload identity pool resource (external account credentials). + If not specified, then it will use a + google.auth.transport.requests.Request client to make requests. Returns: Tuple[google.auth.credentials.Credentials, Optional[str]]: Loaded credentials and the project ID. Authorized user credentials do not - have the project ID information. + have the project ID information. External account credentials project + IDs may not always be determined. Raises: google.auth.exceptions.DefaultCredentialsError: if the file is in the @@ -142,6 +151,14 @@ def load_credentials_from_file(filename, scopes=None, quota_project_id=None): credentials = credentials.with_quota_project(quota_project_id) return credentials, info.get("project_id") + elif credential_type == _EXTERNAL_ACCOUNT_TYPE: + credentials, project_id = _get_external_account_credentials( + info, filename, scopes=scopes, request=request + ) + if quota_project_id: + credentials = credentials.with_quota_project(quota_project_id) + return credentials, project_id + else: raise exceptions.DefaultCredentialsError( "The file {file} does not have a valid type. " @@ -172,7 +189,7 @@ def _get_gcloud_sdk_credentials(): return credentials, project_id -def _get_explicit_environ_credentials(): +def _get_explicit_environ_credentials(request=None): """Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable.""" explicit_file = os.environ.get(environment_vars.CREDENTIALS) @@ -183,7 +200,7 @@ def _get_explicit_environ_credentials(): if explicit_file is not None: credentials, project_id = load_credentials_from_file( - os.environ[environment_vars.CREDENTIALS] + os.environ[environment_vars.CREDENTIALS], request=request ) return credentials, project_id @@ -248,6 +265,57 @@ def _get_gce_credentials(request=None): return None, None +def _get_external_account_credentials(info, filename, scopes=None, request=None): + """Loads external account Credentials from the parsed external account info. + + The credentials information must correspond to a supported external account + credentials. + + Args: + info (Mapping[str, str]): The external account info in Google format. + filename (str): The full path to the credentials file. + scopes (Optional[Sequence[str]]): The list of scopes for the credentials. If + specified, the credentials will automatically be scoped if + necessary. + request (Optional[google.auth.transport.Request]): An object used to make + HTTP requests. This is used to determine the associated project ID + for a workload identity pool resource (external account credentials). + If not specified, then it will use a + google.auth.transport.requests.Request client to make requests. + + Returns: + Tuple[google.auth.credentials.Credentials, Optional[str]]: Loaded + credentials and the project ID. External account credentials project + IDs may not always be determined. + + Raises: + google.auth.exceptions.DefaultCredentialsError: if the info dictionary + is in the wrong format or is missing required information. + """ + # There are currently 2 types of external_account credentials. + try: + # Check if configuration corresponds to an Identity Pool credentials. + from google.auth import identity_pool + + credentials = identity_pool.Credentials.from_info(info, scopes=scopes) + except ValueError: + try: + # Check if configuration corresponds to an AWS credentials. + from google.auth import aws + + credentials = aws.Credentials.from_info(info, scopes=scopes) + except ValueError: + # If the configuration is invalid or does not correspond to any + # supported external_account credentials, raise an error. + raise exceptions.DefaultCredentialsError( + "Failed to load external account credentials from {}".format(filename) + ) + if request is None: + request = google.auth.transport.requests.Request() + + return credentials, credentials.get_project_id(request=request) + + def default(scopes=None, request=None, quota_project_id=None): """Gets the default credentials for the current environment. @@ -261,6 +329,15 @@ def default(scopes=None, request=None, quota_project_id=None): loaded and returned. The project ID returned is the project ID defined in the service account file if available (some older files do not contain project ID information). + + If the environment variable is set to the path of a valid external + account JSON configuration file, then the configuration file is used to + determine and retrieve the external credentials from the current + environment (AWS, Azure, etc). + These will then be exchanged for Google access tokens via the Google STS + endpoint. + The project ID returned in this case is the one corresponding to the + underlying workload identity pool resource if determinable. 2. If the `Google Cloud SDK`_ is installed and has application default credentials set they are loaded and returned. @@ -304,11 +381,15 @@ def default(scopes=None, request=None, quota_project_id=None): scopes (Sequence[str]): The list of scopes for the credentials. If specified, the credentials will automatically be scoped if necessary. - request (google.auth.transport.Request): An object used to make - HTTP requests. This is used to detect whether the application - is running on Compute Engine. If not specified, then it will - use the standard library http client to make requests. - quota_project_id (Optional[str]): The project ID used for + request (Optional[google.auth.transport.Request]): An object used to make + HTTP requests. This is used to either detect whether the application + is running on Compute Engine or to determine the associated project + ID for a workload identity pool resource (external account + credentials). If not specified, then it will either use the standard + library http client to make requests for Compute Engine credentials + or a google.auth.transport.requests.Request client for external + account credentials. + quota_project_id (Optional[str]): The project ID used for quota and billing. Returns: Tuple[~google.auth.credentials.Credentials, Optional[str]]: @@ -328,7 +409,7 @@ def default(scopes=None, request=None, quota_project_id=None): ) checkers = ( - _get_explicit_environ_credentials, + lambda: _get_explicit_environ_credentials(request=request), _get_gcloud_sdk_credentials, _get_gae_credentials, lambda: _get_gce_credentials(request), diff --git a/google/auth/aws.py b/google/auth/aws.py index 3a6d53750..546e82016 100644 --- a/google/auth/aws.py +++ b/google/auth/aws.py @@ -359,6 +359,11 @@ def __init__( scopes (Optional[Sequence[str]]): Optional scopes to request during the authorization grant. + Raises: + google.auth.exceptions.RefreshError: If an error is encountered during + access token retrieval logic. + ValueError: For invalid parameters. + .. note:: Typically one of the helper constructors :meth:`from_file` or :meth:`from_info` are used instead of calling the constructor directly. @@ -393,11 +398,9 @@ def __init__( env_id, env_version = (None, None) if env_id != "aws" or self._cred_verification_url is None: - raise exceptions.GoogleAuthError( - "No valid AWS 'credential_source' provided" - ) + raise ValueError("No valid AWS 'credential_source' provided") elif int(env_version or "") != 1: - raise exceptions.GoogleAuthError( + raise ValueError( "aws version '{}' is not supported in the current build.".format( env_version ) @@ -666,7 +669,7 @@ def from_info(cls, info, **kwargs): google.auth.aws.Credentials: The constructed credentials. Raises: - google.auth.exceptions.GoogleAuthError: For invalid parameters. + ValueError: For invalid parameters. """ return cls( audience=info.get("audience"), diff --git a/google/auth/identity_pool.py b/google/auth/identity_pool.py index 4e78f5a30..43df96273 100644 --- a/google/auth/identity_pool.py +++ b/google/auth/identity_pool.py @@ -73,7 +73,7 @@ def __init__( Raises: google.auth.exceptions.RefreshError: If an error is encountered during access token retrieval logic. - google.auth.exceptions.GoogleAuthError: For invalid parameters. + ValueError: For invalid parameters. .. note:: Typically one of the helper constructors :meth:`from_file` or @@ -101,7 +101,7 @@ def __init__( credential_source_format.get("type") or "text" ) if self._credential_source_format_type not in ["text", "json"]: - raise exceptions.GoogleAuthError( + raise ValueError( "Invalid credential_source format '{}'".format( self._credential_source_format_type ) @@ -112,7 +112,7 @@ def __init__( "subject_token_field_name" ) if self._credential_source_field_name is None: - raise exceptions.GoogleAuthError( + raise ValueError( "Missing subject_token_field_name for JSON credential_source format" ) else: @@ -120,7 +120,7 @@ def __init__( else: self._credential_source_file = None if not self._credential_source_file: - raise exceptions.GoogleAuthError("Missing credential_source file") + raise ValueError("Missing credential_source file") @_helpers.copy_docstring(external_account.Credentials) def retrieve_subject_token(self, request): @@ -173,7 +173,7 @@ def from_info(cls, info, **kwargs): credentials. Raises: - google.auth.exceptions.GoogleAuthError: For invalid parameters. + ValueError: For invalid parameters. """ return cls( audience=info.get("audience"), diff --git a/noxfile.py b/noxfile.py index 79a09e4da..f46909c42 100644 --- a/noxfile.py +++ b/noxfile.py @@ -30,7 +30,12 @@ "grpcio", ] -ASYNC_DEPENDENCIES = ["pytest-asyncio", "aioresponses", "asynctest"] +ASYNC_DEPENDENCIES = [ + "pytest-asyncio", + "aiohttp < 3.7.0dev", + "aioresponses", + "asynctest", +] BLACK_VERSION = "black==19.3b0" BLACK_PATHS = [ diff --git a/setup.py b/setup.py index 522b98103..16c277950 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ "six>=1.9.0", ) -extras = {"aiohttp": "aiohttp >= 3.6.2, < 4.0.0dev; python_version>='3.6'"} +extras = {"aiohttp": "aiohttp >= 3.6.2, < 3.7.0dev; python_version>='3.6'"} with io.open("README.rst", "r") as fh: long_description = fh.read() diff --git a/system_tests/noxfile.py b/system_tests/noxfile.py index a039228d9..0f852ea27 100644 --- a/system_tests/noxfile.py +++ b/system_tests/noxfile.py @@ -168,7 +168,7 @@ def configure_cloud_sdk(session, application_default_credentials, project=False) # Test sesssions -TEST_DEPENDENCIES_ASYNC = ["aiohttp", "pytest-asyncio", "nest-asyncio"] +TEST_DEPENDENCIES_ASYNC = ["aiohttp < 3.7.0dev", "pytest-asyncio", "nest-asyncio"] TEST_DEPENDENCIES_SYNC = ["pytest", "requests"] PYTHON_VERSIONS_ASYNC = ["3.7"] PYTHON_VERSIONS_SYNC = ["2.7", "3.7"] diff --git a/tests/test__default.py b/tests/test__default.py index 2738e22bc..de0775946 100644 --- a/tests/test__default.py +++ b/tests/test__default.py @@ -20,10 +20,13 @@ from google.auth import _default from google.auth import app_engine +from google.auth import aws from google.auth import compute_engine from google.auth import credentials from google.auth import environment_vars from google.auth import exceptions +from google.auth import external_account +from google.auth import identity_pool from google.oauth2 import service_account import google.oauth2.credentials @@ -49,6 +52,34 @@ with open(SERVICE_ACCOUNT_FILE) as fh: SERVICE_ACCOUNT_FILE_DATA = json.load(fh) +SUBJECT_TOKEN_TEXT_FILE = os.path.join(DATA_DIR, "external_subject_token.txt") +TOKEN_URL = "https://sts.googleapis.com/v1/token" +AUDIENCE = "//iam.googleapis.com/projects/123456/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID" +REGION_URL = "http://169.254.169.254/latest/meta-data/placement/availability-zone" +SECURITY_CREDS_URL = "http://169.254.169.254/latest/meta-data/iam/security-credentials" +CRED_VERIFICATION_URL = ( + "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15" +) +IDENTITY_POOL_DATA = { + "type": "external_account", + "audience": AUDIENCE, + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": TOKEN_URL, + "credential_source": {"file": SUBJECT_TOKEN_TEXT_FILE}, +} +AWS_DATA = { + "type": "external_account", + "audience": AUDIENCE, + "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request", + "token_url": TOKEN_URL, + "credential_source": { + "environment_id": "aws1", + "region_url": REGION_URL, + "url": SECURITY_CREDS_URL, + "regional_cred_verification_url": CRED_VERIFICATION_URL, + }, +} + MOCK_CREDENTIALS = mock.Mock(spec=credentials.CredentialsWithQuotaProject) MOCK_CREDENTIALS.with_quota_project.return_value = MOCK_CREDENTIALS @@ -57,6 +88,12 @@ return_value=(MOCK_CREDENTIALS, mock.sentinel.project_id), autospec=True, ) +EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH = mock.patch.object( + external_account.Credentials, + "get_project_id", + return_value=mock.sentinel.project_id, + autospec=True, +) def test_load_credentials_from_missing_file(): @@ -185,6 +222,87 @@ def test_load_credentials_from_file_service_account_bad_format(tmpdir): assert excinfo.match(r"missing fields") +@EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH +def test_load_credentials_from_file_external_account_identity_pool( + get_project_id, tmpdir +): + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(IDENTITY_POOL_DATA)) + credentials, project_id = _default.load_credentials_from_file(str(config_file)) + + assert isinstance(credentials, identity_pool.Credentials) + assert project_id is mock.sentinel.project_id + assert get_project_id.called + + +@EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH +def test_load_credentials_from_file_external_account_aws(get_project_id, tmpdir): + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(AWS_DATA)) + credentials, project_id = _default.load_credentials_from_file(str(config_file)) + + assert isinstance(credentials, aws.Credentials) + assert project_id is mock.sentinel.project_id + assert get_project_id.called + + +@EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH +def test_load_credentials_from_file_external_account_with_scopes( + get_project_id, tmpdir +): + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(IDENTITY_POOL_DATA)) + credentials, project_id = _default.load_credentials_from_file( + str(config_file), scopes=["https://www.google.com/calendar/feeds"] + ) + + assert isinstance(credentials, identity_pool.Credentials) + assert project_id is mock.sentinel.project_id + assert credentials.scopes == ["https://www.google.com/calendar/feeds"] + + +@EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH +def test_load_credentials_from_file_external_account_with_quota_project( + get_project_id, tmpdir +): + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(IDENTITY_POOL_DATA)) + credentials, project_id = _default.load_credentials_from_file( + str(config_file), quota_project_id="project-foo" + ) + + assert isinstance(credentials, identity_pool.Credentials) + assert project_id is mock.sentinel.project_id + assert credentials.quota_project_id == "project-foo" + + +def test_load_credentials_from_file_external_account_bad_format(tmpdir): + filename = tmpdir.join("external_account_bad.json") + filename.write(json.dumps({"type": "external_account"})) + + with pytest.raises(exceptions.DefaultCredentialsError) as excinfo: + _default.load_credentials_from_file(str(filename)) + + assert excinfo.match( + "Failed to load external account credentials from {}".format(str(filename)) + ) + + +@EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH +def test_load_credentials_from_file_external_account_explicit_request( + get_project_id, tmpdir +): + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(IDENTITY_POOL_DATA)) + credentials, project_id = _default.load_credentials_from_file( + str(config_file), request=mock.sentinel.request + ) + + assert isinstance(credentials, identity_pool.Credentials) + assert project_id is mock.sentinel.project_id + get_project_id.assert_called_with(credentials, request=mock.sentinel.request) + + @mock.patch.dict(os.environ, {}, clear=True) def test__get_explicit_environ_credentials_no_env(): assert _default._get_explicit_environ_credentials() == (None, None) @@ -198,7 +316,7 @@ def test__get_explicit_environ_credentials(load, monkeypatch): assert credentials is MOCK_CREDENTIALS assert project_id is mock.sentinel.project_id - load.assert_called_with("filename") + load.assert_called_with("filename", request=None) @LOAD_FILE_PATCH @@ -503,3 +621,65 @@ def test_default_no_app_engine_compute_engine_module(unused_get): sys.modules["google.auth.compute_engine"] = None sys.modules["google.auth.app_engine"] = None assert _default.default() == (MOCK_CREDENTIALS, mock.sentinel.project_id) + + +@EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH +def test_default_environ_external_credentials(get_project_id, monkeypatch, tmpdir): + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(IDENTITY_POOL_DATA)) + monkeypatch.setenv(environment_vars.CREDENTIALS, str(config_file)) + + credentials, project_id = _default.default() + + assert isinstance(credentials, identity_pool.Credentials) + assert project_id is mock.sentinel.project_id + + +@EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH +def test_default_environ_external_credentials_with_scopes_and_quota_project_id( + get_project_id, monkeypatch, tmpdir +): + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(IDENTITY_POOL_DATA)) + monkeypatch.setenv(environment_vars.CREDENTIALS, str(config_file)) + + credentials, project_id = _default.default( + scopes=["https://www.google.com/calendar/feeds"], quota_project_id="project-foo" + ) + + assert isinstance(credentials, identity_pool.Credentials) + assert project_id is mock.sentinel.project_id + assert credentials.quota_project_id == "project-foo" + assert credentials.scopes == ["https://www.google.com/calendar/feeds"] + + +@EXTERNAL_ACCOUNT_GET_PROJECT_ID_PATCH +def test_default_environ_external_credentials_explicit_request( + get_project_id, monkeypatch, tmpdir +): + config_file = tmpdir.join("config.json") + config_file.write(json.dumps(IDENTITY_POOL_DATA)) + monkeypatch.setenv(environment_vars.CREDENTIALS, str(config_file)) + + credentials, project_id = _default.default(request=mock.sentinel.request) + + assert isinstance(credentials, identity_pool.Credentials) + assert project_id is mock.sentinel.project_id + # default() will initialize new credentials via with_scopes_if_required + # and potentially with_quota_project. + # As a result the caller of get_project_id() will not match the returned + # credentials. + get_project_id.assert_called_with(mock.ANY, request=mock.sentinel.request) + + +def test_default_environ_external_credentials_bad_format(monkeypatch, tmpdir): + filename = tmpdir.join("external_account_bad.json") + filename.write(json.dumps({"type": "external_account"})) + monkeypatch.setenv(environment_vars.CREDENTIALS, str(filename)) + + with pytest.raises(exceptions.DefaultCredentialsError) as excinfo: + _default.default() + + assert excinfo.match( + "Failed to load external account credentials from {}".format(str(filename)) + ) diff --git a/tests/test_aws.py b/tests/test_aws.py index 5322c32c6..5d18f7ce1 100644 --- a/tests/test_aws.py +++ b/tests/test_aws.py @@ -837,7 +837,7 @@ def test_constructor_invalid_credential_source(self): # Provide invalid credential source. credential_source = {"unsupported": "value"} - with pytest.raises(exceptions.GoogleAuthError) as excinfo: + with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source=credential_source) assert excinfo.match(r"No valid AWS 'credential_source' provided") @@ -847,7 +847,7 @@ def test_constructor_invalid_environment_id(self): credential_source = self.CREDENTIAL_SOURCE.copy() credential_source["environment_id"] = "azure1" - with pytest.raises(exceptions.GoogleAuthError) as excinfo: + with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source=credential_source) assert excinfo.match(r"No valid AWS 'credential_source' provided") @@ -857,7 +857,7 @@ def test_constructor_missing_cred_verification_url(self): credential_source = self.CREDENTIAL_SOURCE.copy() credential_source.pop("regional_cred_verification_url") - with pytest.raises(exceptions.GoogleAuthError) as excinfo: + with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source=credential_source) assert excinfo.match(r"No valid AWS 'credential_source' provided") @@ -867,7 +867,7 @@ def test_constructor_invalid_environment_id_version(self): credential_source = self.CREDENTIAL_SOURCE.copy() credential_source["environment_id"] = "aws3" - with pytest.raises(exceptions.GoogleAuthError) as excinfo: + with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source=credential_source) assert excinfo.match(r"aws version '3' is not supported in the current build.") diff --git a/tests/test_identity_pool.py b/tests/test_identity_pool.py index 26da3804a..47a6fbdb3 100644 --- a/tests/test_identity_pool.py +++ b/tests/test_identity_pool.py @@ -349,13 +349,13 @@ def test_from_file_required_options_only(self, mock_init, tmpdir): def test_constructor_invalid_options(self): credential_source = {"unsupported": "value"} - with pytest.raises(exceptions.GoogleAuthError) as excinfo: + with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source=credential_source) assert excinfo.match(r"Missing credential_source file") def test_constructor_invalid_credential_source(self): - with pytest.raises(exceptions.GoogleAuthError) as excinfo: + with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source="non-dict") assert excinfo.match(r"Missing credential_source file") @@ -363,7 +363,7 @@ def test_constructor_invalid_credential_source(self): def test_constructor_invalid_credential_source_format_type(self): credential_source = {"format": {"type": "xml"}} - with pytest.raises(exceptions.GoogleAuthError) as excinfo: + with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source=credential_source) assert excinfo.match(r"Invalid credential_source format 'xml'") @@ -371,7 +371,7 @@ def test_constructor_invalid_credential_source_format_type(self): def test_constructor_missing_subject_token_field_name(self): credential_source = {"format": {"type": "json"}} - with pytest.raises(exceptions.GoogleAuthError) as excinfo: + with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source=credential_source) assert excinfo.match(