diff --git a/docs/changelog/142021.yaml b/docs/changelog/142021.yaml new file mode 100644 index 0000000000000..ccbb5db1a963b --- /dev/null +++ b/docs/changelog/142021.yaml @@ -0,0 +1,5 @@ +area: SQL +issues: [] +pr: 142021 +summary: Add support for API key to JDBC and CLI +type: enhancement diff --git a/docs/reference/query-languages/sql/sql-cli.md b/docs/reference/query-languages/sql/sql-cli.md index 39036a9d01dff..9e496d878ee4b 100644 --- a/docs/reference/query-languages/sql/sql-cli.md +++ b/docs/reference/query-languages/sql/sql-cli.md @@ -28,6 +28,22 @@ If security is enabled on your cluster, you can pass the username and password i $ ./bin/elasticsearch-sql-cli https://sql_user:strongpassword@some.server:9200 ``` +### API Key Authentication [sql-cli-apikey] + +As an alternative to basic authentication, you can use API key authentication with the `--apikey` option. API keys can be created using the [Create API key API](docs-content://deploy-manage/api-keys/elasticsearch-api-keys.md). The API key should be provided in its encoded form (the `encoded` value returned by the Create API key API): + +```bash +$ ./bin/elasticsearch-sql-cli --apikey https://some.server:9200 +``` + +::::{note} +When using API key authentication, do not include username and password in the URL. The CLI will return an error if both API key and basic authentication credentials are provided. +:::: + +::::{warning} +Command line arguments are visible to other users on the system through process listing commands like `ps aux` or by inspecting `/proc//cmdline`. Avoid using this method on shared systems where other users might be able to view your credentials. +:::: + Once the CLI is running you can use any [query](elasticsearch://reference/query-languages/sql/sql-spec.md) that Elasticsearch supports: ```sql diff --git a/docs/reference/query-languages/sql/sql-jdbc.md b/docs/reference/query-languages/sql/sql-jdbc.md index ad2bbc20c457b..f8a3f44f5e600 100644 --- a/docs/reference/query-languages/sql/sql-jdbc.md +++ b/docs/reference/query-languages/sql/sql-jdbc.md @@ -120,6 +120,24 @@ $$$jdbc-cfg-timezone$$$ : Basic Authentication password +### API Key Authentication [jdbc-cfg-auth-apikey] + +As an alternative to basic authentication, you can use API key authentication. API keys can be created using the [Create API key API](docs-content://deploy-manage/api-keys/elasticsearch-api-keys.md). The API key should be provided in its encoded form (the `encoded` value returned by the Create API key API). + +`apiKey` +: Encoded API key for authentication. Cannot be used together with `user`/`password` basic authentication. + +::::{note} +When using API key authentication, do not specify `user` or `password`. The driver will return an error if both API key and basic authentication credentials are provided. +:::: + +Example connection URL using API key: + +```text +jdbc:es://http://server:9200/?apiKey= +``` + + ### SSL [jdbc-cfg-ssl] `ssl` (default `false`) diff --git a/docs/reference/query-languages/sql/sql-security.md b/docs/reference/query-languages/sql/sql-security.md index 87dcb2514c9bc..45f9585ed5eb2 100644 --- a/docs/reference/query-languages/sql/sql-security.md +++ b/docs/reference/query-languages/sql/sql-security.md @@ -20,11 +20,14 @@ In case of an encrypted transport, the SSL/TLS support needs to be enabled in El ## Authentication [_authentication] -The authentication support in {{es}} SQL is of two types: +The authentication support in {{es}} SQL is of three types: Username/Password : Set these through `user` and `password` properties. +API Key +: Use an API key for authentication by setting the `apiKey` property. API keys can be created using the [Create API key API](docs-content://deploy-manage/api-keys/elasticsearch-api-keys.md). The API key should be provided in its encoded form (the `encoded` value returned by the Create API key API). This is an alternative to username/password authentication and cannot be used together with it. For the CLI, use the `--apikey` command line option. + PKI/X.509 : Use X.509 certificates to authenticate {{es}} SQL to {{es}}. For this, one would need to setup the `keystore` containing the private key and certificate to the appropriate user (configured in {{es}}) and the `truststore` with the CA certificate used to sign the SSL/TLS certificates in the {{es}} cluster. That is, one should setup the key to authenticate {{es}} SQL and also to verify that is the right one. To do so, one should set the `ssl.keystore.location` and `ssl.truststore.location` properties to indicate the `keystore` and `truststore` to use. It is recommended to have these secured through a password in which case `ssl.keystore.pass` and `ssl.truststore.pass` properties are required. diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/Command.java b/libs/cli/src/main/java/org/elasticsearch/cli/Command.java index 1690515532e7b..ae25d569acd47 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/Command.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/Command.java @@ -89,13 +89,19 @@ protected void mainWithoutErrorHandling(String[] args, Terminal terminal, Proces LoggerFactory loggerFactory = LoggerFactory.provider(); if (options.has(silentOption)) { terminal.setVerbosity(Terminal.Verbosity.SILENT); - loggerFactory.setRootLevel(Level.OFF); + if (loggerFactory != null) { + loggerFactory.setRootLevel(Level.OFF); + } } else if (options.has(verboseOption)) { terminal.setVerbosity(Terminal.Verbosity.VERBOSE); - loggerFactory.setRootLevel(Level.DEBUG); + if (loggerFactory != null) { + loggerFactory.setRootLevel(Level.DEBUG); + } } else { terminal.setVerbosity(Terminal.Verbosity.NORMAL); - loggerFactory.setRootLevel(Level.INFO); + if (loggerFactory != null) { + loggerFactory.setRootLevel(Level.INFO); + } } execute(terminal, options, processInfo); diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java index cfa926055d3d3..bc2c38fdb07f3 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java @@ -25,6 +25,8 @@ import java.util.Properties; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.AUTH_API_KEY; +import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.AUTH_USER; import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.CONNECT_TIMEOUT; import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.NETWORK_TIMEOUT; import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.PAGE_SIZE; @@ -33,6 +35,7 @@ import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.QUERY_TIMEOUT; import static org.elasticsearch.xpack.sql.jdbc.JdbcConfiguration.URL_FULL_PREFIX; import static org.elasticsearch.xpack.sql.jdbc.JdbcConfiguration.URL_PREFIX; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -399,4 +402,40 @@ private void assertJdbcSqlException(String wrongSetting, String correctSetting, JdbcSQLException ex = expectThrows(JdbcSQLException.class, () -> JdbcConfiguration.create(url, props, 0)); assertEquals("Unknown parameter [" + wrongSetting + "]; did you mean [" + correctSetting + "]", ex.getMessage()); } + + public void testApiKeyInUrl() throws Exception { + String apiKey = "test_api_key_encoded"; + JdbcConfiguration ci = ci(jdbcPrefix() + "test:9200?apiKey=" + apiKey); + assertThat(ci.apiKey(), is(apiKey)); + assertNull(ci.authUser()); + assertNull(ci.authPass()); + } + + public void testApiKeyInProperties() throws Exception { + String apiKey = "test_api_key_encoded"; + Properties props = new Properties(); + props.setProperty(AUTH_API_KEY, apiKey); + JdbcConfiguration ci = JdbcConfiguration.create(jdbcPrefix() + "test:9200", props, 0); + assertThat(ci.apiKey(), is(apiKey)); + assertNull(ci.authUser()); + assertNull(ci.authPass()); + } + + public void testApiKeyAndUserMutuallyExclusive() { + String apiKey = "test_api_key_encoded"; + Properties props = new Properties(); + props.setProperty(AUTH_API_KEY, apiKey); + props.setProperty(AUTH_USER, "user"); + JdbcSQLException ex = expectThrows(JdbcSQLException.class, () -> JdbcConfiguration.create(jdbcPrefix() + "test:9200", props, 0)); + assertThat(ex.getMessage(), containsString("Cannot use both API key and basic authentication")); + } + + public void testApiKeyAndUserInUrlMutuallyExclusive() { + String apiKey = "test_api_key_encoded"; + JdbcSQLException ex = expectThrows( + JdbcSQLException.class, + () -> ci(jdbcPrefix() + "test:9200?apiKey=" + apiKey + "&user=testuser") + ); + assertThat(ex.getMessage(), containsString("Cannot use both API key and basic authentication")); + } } diff --git a/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliApiKeyIT.java b/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliApiKeyIT.java new file mode 100644 index 0000000000000..ff9c04e87e76c --- /dev/null +++ b/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliApiKeyIT.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.sql.qa.security; + +import org.elasticsearch.client.Request; +import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli; +import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli.ApiKeySecurityConfig; + +import static org.elasticsearch.xpack.sql.qa.security.RestSqlIT.SSL_ENABLED; +import static org.hamcrest.Matchers.containsString; + +/** + * Integration tests for CLI connections using API key authentication. + */ +public class CliApiKeyIT extends SqlApiKeyTestCase { + + public void testCliConnectionWithApiKey() throws Exception { + String encodedApiKey = createApiKey(""" + { + "name": "cli_test_key", + "role_descriptors": { + "role": { + "cluster": ["monitor"], + "indices": [ + { + "names": ["*"], + "privileges": ["all"] + } + ] + } + } + } + """); + + Request createIndex = new Request("PUT", "/test_cli_api_key"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "value": { "type": "integer" } + } + } + } + """); + client().performRequest(createIndex); + + Request indexDoc = new Request("PUT", "/test_cli_api_key/_doc/1"); + indexDoc.addParameter("refresh", "true"); + indexDoc.setJsonEntity(""" + { + "value": 123 + } + """); + client().performRequest(indexDoc); + + ApiKeySecurityConfig apiKeyConfig = createApiKeySecurityConfig(encodedApiKey); + + try (EmbeddedCli cli = new EmbeddedCli(elasticsearchAddress(), true, apiKeyConfig)) { + String result = cli.command("SELECT value FROM test_cli_api_key"); + assertThat(result, containsString("value")); + cli.readLine(); // separator line + String valueLine = cli.readLine(); + assertThat(valueLine, containsString("123")); + } + } + + public void testCliConnectionWithInvalidApiKey() throws Exception { + ApiKeySecurityConfig apiKeyConfig = new ApiKeySecurityConfig( + SSL_ENABLED, + "invalid_api_key_value", + SSL_ENABLED ? SqlSecurityTestCluster.getKeystorePath() : null, + SSL_ENABLED ? SqlSecurityTestCluster.KEYSTORE_PASSWORD : null + ); + + try (EmbeddedCli cli = new EmbeddedCli(elasticsearchAddress(), false, apiKeyConfig)) { + String result = cli.command("SELECT 1"); + StringBuilder fullError = new StringBuilder(result); + String line; + while ((line = cli.readLine()) != null && !line.isEmpty()) { + fullError.append(line); + } + String errorMessage = fullError.toString(); + assertTrue( + "Expected authentication error but got: " + errorMessage, + errorMessage.contains("security_exception") || errorMessage.contains("Communication error") + ); + } + } + + public void testCliConnectionWithLimitedApiKey() throws Exception { + Request createRestrictedIndex = new Request("PUT", "/cli_restricted_index"); + createRestrictedIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "secret": { "type": "keyword" } + } + } + } + """); + client().performRequest(createRestrictedIndex); + + Request indexRestrictedDoc = new Request("PUT", "/cli_restricted_index/_doc/1"); + indexRestrictedDoc.addParameter("refresh", "true"); + indexRestrictedDoc.setJsonEntity(""" + { + "secret": "confidential" + } + """); + client().performRequest(indexRestrictedDoc); + + String encodedApiKey = createApiKey(""" + { + "name": "cli_limited_key", + "role_descriptors": { + "role": { + "cluster": ["monitor"], + "indices": [ + { + "names": ["cli_allowed_*"], + "privileges": ["read"] + } + ] + } + } + } + """); + + ApiKeySecurityConfig apiKeyConfig = createApiKeySecurityConfig(encodedApiKey); + + try (EmbeddedCli cli = new EmbeddedCli(elasticsearchAddress(), true, apiKeyConfig)) { + String result = cli.command("SELECT * FROM cli_restricted_index"); + String errorLine = cli.readLine(); + assertThat(errorLine, containsString("Unknown index [cli_restricted_index]")); + } + } + + private ApiKeySecurityConfig createApiKeySecurityConfig(String apiKey) { + return new ApiKeySecurityConfig( + SSL_ENABLED, + apiKey, + SSL_ENABLED ? SqlSecurityTestCluster.getKeystorePath() : null, + SSL_ENABLED ? SqlSecurityTestCluster.KEYSTORE_PASSWORD : null + ); + } +} diff --git a/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcApiKeyIT.java b/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcApiKeyIT.java new file mode 100644 index 0000000000000..d4ca93398ce6d --- /dev/null +++ b/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcApiKeyIT.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.sql.qa.security; + +import org.elasticsearch.client.Request; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import static org.elasticsearch.xpack.sql.qa.security.RestSqlIT.SSL_ENABLED; +import static org.hamcrest.Matchers.containsString; + +/** + * Integration tests for JDBC connections using API key authentication. + */ +public class JdbcApiKeyIT extends SqlApiKeyTestCase { + + public void testJdbcConnectionWithApiKey() throws Exception { + String encodedApiKey = createApiKey(""" + { + "name": "jdbc_test_key", + "role_descriptors": { + "role": { + "cluster": ["monitor"], + "indices": [ + { + "names": ["*"], + "privileges": ["all"] + } + ] + } + } + } + """); + + Request createIndex = new Request("PUT", "/test_api_key"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "value": { "type": "integer" } + } + } + } + """); + client().performRequest(createIndex); + + Request indexDoc = new Request("PUT", "/test_api_key/_doc/1"); + indexDoc.addParameter("refresh", "true"); + indexDoc.setJsonEntity(""" + { + "value": 42 + } + """); + client().performRequest(indexDoc); + + Properties props = createJdbcPropertiesWithApiKey(encodedApiKey); + String jdbcUrl = jdbcUrl(); + + try (Connection connection = DriverManager.getConnection(jdbcUrl, props)) { + try (Statement statement = connection.createStatement()) { + try (ResultSet rs = statement.executeQuery("SELECT value FROM test_api_key")) { + assertTrue("Expected at least one result", rs.next()); + assertEquals(42, rs.getInt("value")); + assertFalse("Expected only one result", rs.next()); + } + } + } + } + + public void testJdbcConnectionWithInvalidApiKey() throws Exception { + Properties props = createJdbcPropertiesWithApiKey("invalid_api_key_value"); + String jdbcUrl = jdbcUrl(); + + SQLException e = expectThrows(SQLException.class, () -> { + try (Connection connection = DriverManager.getConnection(jdbcUrl, props)) { + connection.createStatement().executeQuery("SELECT 1"); + } + }); + assertThat(e.getMessage(), containsString("security_exception")); + } + + public void testJdbcConnectionWithLimitedApiKey() throws Exception { + Request createRestrictedIndex = new Request("PUT", "/restricted_index"); + createRestrictedIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "secret": { "type": "keyword" } + } + } + } + """); + client().performRequest(createRestrictedIndex); + + Request indexRestrictedDoc = new Request("PUT", "/restricted_index/_doc/1"); + indexRestrictedDoc.addParameter("refresh", "true"); + indexRestrictedDoc.setJsonEntity(""" + { + "secret": "confidential" + } + """); + client().performRequest(indexRestrictedDoc); + + String encodedApiKey = createApiKey(""" + { + "name": "limited_key", + "role_descriptors": { + "role": { + "cluster": ["monitor"], + "indices": [ + { + "names": ["allowed_*"], + "privileges": ["read"] + } + ] + } + } + } + """); + + Properties props = createJdbcPropertiesWithApiKey(encodedApiKey); + String jdbcUrl = jdbcUrl(); + + try (Connection connection = DriverManager.getConnection(jdbcUrl, props)) { + try (Statement statement = connection.createStatement()) { + SQLException e = expectThrows(SQLException.class, () -> statement.executeQuery("SELECT * FROM restricted_index")); + assertThat(e.getMessage(), containsString("Unknown index [restricted_index]")); + } + } + } + + private String jdbcUrl() { + return "jdbc:es://" + getProtocol() + "://" + elasticsearchAddress(); + } + + private Properties createJdbcPropertiesWithApiKey(String apiKey) { + Properties props = new Properties(); + props.setProperty("apiKey", apiKey); + props.setProperty("timezone", "UTC"); + addSslPropertiesIfNeeded(props); + return props; + } + + private void addSslPropertiesIfNeeded(Properties properties) { + if (SSL_ENABLED == false) { + return; + } + String keystorePath = SqlSecurityTestCluster.getKeystorePath(); + String keystorePass = SqlSecurityTestCluster.KEYSTORE_PASSWORD; + + properties.put("ssl", "true"); + properties.put("ssl.keystore.location", keystorePath); + properties.put("ssl.keystore.pass", keystorePass); + properties.put("ssl.truststore.location", keystorePath); + properties.put("ssl.truststore.pass", keystorePass); + } +} diff --git a/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlApiKeyTestCase.java b/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlApiKeyTestCase.java new file mode 100644 index 0000000000000..075307d6c829b --- /dev/null +++ b/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlApiKeyTestCase.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.sql.qa.security; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.After; +import org.junit.ClassRule; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.sql.qa.security.RestSqlIT.SSL_ENABLED; + +/** + * Base class for integration tests that use API key authentication. + */ +public abstract class SqlApiKeyTestCase extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = SqlSecurityTestCluster.getCluster(); + + private final List createdApiKeyIds = new ArrayList<>(); + + @After + public void cleanupApiKeys() throws IOException { + for (String apiKeyId : createdApiKeyIds) { + try { + Request deleteApiKey = new Request("DELETE", "/_security/api_key"); + deleteApiKey.setJsonEntity("{\"ids\": [\"" + apiKeyId + "\"]}"); + client().performRequest(deleteApiKey); + } catch (Exception e) { + logger.warn("Failed to delete API key [{}]: {}", apiKeyId, e.getMessage()); + } + } + createdApiKeyIds.clear(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected Settings restClientSettings() { + return RestSqlIT.securitySettings(); + } + + @Override + protected String getProtocol() { + return SSL_ENABLED ? "https" : "http"; + } + + protected String elasticsearchAddress() { + return getTestRestCluster().split(",")[0]; + } + + protected String createApiKey(String body) throws IOException { + Request createApiKey = new Request("POST", "/_security/api_key"); + createApiKey.setJsonEntity(body); + Response response = client().performRequest(createApiKey); + + try (InputStream content = response.getEntity().getContent()) { + Map responseMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); + String apiKeyId = (String) responseMap.get("id"); + createdApiKeyIds.add(apiKeyId); + return (String) responseMap.get("encoded"); + } + } +} diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/cli/EmbeddedCli.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/cli/EmbeddedCli.java index d40d380fecd34..34d7f906f34ca 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/cli/EmbeddedCli.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/cli/EmbeddedCli.java @@ -66,9 +66,26 @@ public class EmbeddedCli implements Closeable { */ private boolean closed = false; - @SuppressWarnings("this-escape") public EmbeddedCli(String elasticsearchAddress, boolean checkConnectionOnStartup, @Nullable SecurityConfig security) throws IOException { + this(elasticsearchAddress, checkConnectionOnStartup, security, null); + } + + /** + * Create an embedded CLI with API key authentication. + */ + public EmbeddedCli(String elasticsearchAddress, boolean checkConnectionOnStartup, ApiKeySecurityConfig apiKeySecurity) + throws IOException { + this(elasticsearchAddress, checkConnectionOnStartup, null, apiKeySecurity); + } + + @SuppressWarnings("this-escape") + private EmbeddedCli( + String elasticsearchAddress, + boolean checkConnectionOnStartup, + @Nullable SecurityConfig security, + @Nullable ApiKeySecurityConfig apiKeySecurity + ) throws IOException { PipedOutputStream outgoing = new PipedOutputStream(); PipedInputStream cliIn = new PipedInputStream(outgoing); PipedInputStream incoming = new PipedInputStream(); @@ -82,9 +99,25 @@ public EmbeddedCli(String elasticsearchAddress, boolean checkConnectionOnStartup in = new BufferedReader(new InputStreamReader(incoming, StandardCharsets.UTF_8)); List args = new ArrayList<>(); - if (security == null) { + if (security == null && apiKeySecurity == null) { args.add(elasticsearchAddress); + } else if (apiKeySecurity != null) { + // API key authentication + String address = elasticsearchAddress; + if (apiKeySecurity.https) { + address = "https://" + address; + } else if (randomBoolean()) { + address = "http://" + address; + } + args.add(address); + args.add("-apikey"); + args.add(apiKeySecurity.apiKey); + if (apiKeySecurity.keystoreLocation != null) { + args.add("-keystore_location"); + args.add(apiKeySecurity.keystoreLocation); + } } else { + // Basic authentication String address = security.user + "@" + elasticsearchAddress; if (security.https) { address = "https://" + address; @@ -126,7 +159,7 @@ public EmbeddedCli(String elasticsearchAddress, boolean checkConnectionOnStartup exec.start(); try { - // Feed it passwords if needed + // Feed it passwords if needed (only for basic auth, not API key) if (security != null) { String passwordPrompt = "[?1h=[?2004hpassword: "; if (security.keystoreLocation != null) { @@ -155,6 +188,15 @@ public EmbeddedCli(String elasticsearchAddress, boolean checkConnectionOnStartup logger.info("out: {}", security.password); // Read the newline echoed after the password prompt assertEquals("", readLine()); + } else if (apiKeySecurity != null && apiKeySecurity.keystoreLocation != null) { + // For API key auth with SSL, we still need to provide keystore password + assertEquals("[?1h=[?2004hkeystore password: ", readUntil(s -> s.endsWith(": "))); + out.write(apiKeySecurity.keystorePassword + "\n"); + out.flush(); + logger.info("out: {}", apiKeySecurity.keystorePassword); + // Read the newline echoed after the password prompt + assertEquals("", readLine()); + assertEquals("", readLine()); } // Read until the first "good" line (skip the logo or read until an exception) @@ -377,4 +419,44 @@ public String keystorePassword() { return keystorePassword; } } + + /** + * Configuration for API key authentication. + */ + public static class ApiKeySecurityConfig { + private final boolean https; + private final String apiKey; + @Nullable + private final String keystoreLocation; + @Nullable + private final String keystorePassword; + + public ApiKeySecurityConfig(boolean https, String apiKey, @Nullable String keystoreLocation, @Nullable String keystorePassword) { + if (apiKey == null) { + throw new IllegalArgumentException("[apiKey] is required."); + } + if (keystoreLocation == null) { + if (keystorePassword != null) { + throw new IllegalArgumentException("[keystorePassword] cannot be specified if [keystoreLocation] is not specified"); + } + } else { + if (keystorePassword == null) { + throw new IllegalArgumentException("[keystorePassword] is required if [keystoreLocation] is specified"); + } + } + + this.https = https; + this.apiKey = apiKey; + this.keystoreLocation = keystoreLocation; + this.keystorePassword = keystorePassword; + } + + public String keystoreLocation() { + return keystoreLocation; + } + + public String keystorePassword() { + return keystorePassword; + } + } } diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java index 8bd1e2c5d0581..5c44afe101a58 100644 --- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java +++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/Cli.java @@ -40,6 +40,7 @@ public class Cli extends Command { private final OptionSpec keystoreLocation; + private final OptionSpec apiKeyOption; private final OptionSpec checkOption; private final OptionSpec connectionString; private final OptionSpec binaryCommunication; @@ -101,6 +102,12 @@ public Cli(CliTerminal cliTerminal) { + "If specified then the CLI will prompt for a keystore password. " + "If specified when the uri isn't https then an error is thrown." ).withRequiredArg().ofType(String.class); + this.apiKeyOption = parser.acceptsAll( + Arrays.asList("apikey"), + "API key to use for authentication. " + + "The API key should be in the encoded format (base64 of id:api_key). " + + "Cannot be used together with basic authentication (user in the URI)." + ).withRequiredArg().ofType(String.class); this.checkOption = parser.acceptsAll(Arrays.asList("c", "check"), "Enable initial connection check on startup") .withRequiredArg() .ofType(Boolean.class) @@ -123,10 +130,16 @@ protected void execute(Terminal terminal, OptionSet options, ProcessInfo process throw new UserException(ExitCodes.USAGE, "expecting a single keystore file"); } String keystoreLocationValue = args.size() == 1 ? args.get(0) : null; - execute(uri, debug, binary, keystoreLocationValue, checkConnection); + args = apiKeyOption.values(options); + if (args.size() > 1) { + throw new UserException(ExitCodes.USAGE, "expecting a single API key"); + } + String apiKey = args.size() == 1 ? args.get(0) : null; + execute(uri, debug, binary, keystoreLocationValue, checkConnection, apiKey); } - private void execute(String uri, boolean debug, boolean binary, String keystoreLocation, boolean checkConnection) throws Exception { + private void execute(String uri, boolean debug, boolean binary, String keystoreLocation, boolean checkConnection, String apiKey) + throws Exception { CliCommand cliCommand = new CliCommands( new PrintLogoCommand(), new ClearScreenCliCommand(), @@ -139,7 +152,7 @@ private void execute(String uri, boolean debug, boolean binary, String keystoreL ); try { ConnectionBuilder connectionBuilder = new ConnectionBuilder(cliTerminal); - ConnectionConfiguration con = connectionBuilder.buildConnection(uri, keystoreLocation, binary); + ConnectionConfiguration con = connectionBuilder.buildConnection(uri, keystoreLocation, binary, apiKey); CliSession cliSession = new CliSession(new HttpClient(con)); cliSession.cfg().setDebug(debug); if (checkConnection) { diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilder.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilder.java index 9886d3b2637d9..8ded55d993046 100644 --- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilder.java +++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilder.java @@ -43,6 +43,24 @@ public ConnectionBuilder(CliTerminal cliTerminal) { */ public ConnectionConfiguration buildConnection(String connectionStringArg, String keystoreLocation, boolean binaryCommunication) throws UserException { + return buildConnection(connectionStringArg, keystoreLocation, binaryCommunication, null); + } + + /** + * Build the connection. + * + * @param connectionStringArg the connection string to connect to + * @param keystoreLocation the location of the keystore to configure. If null then use the system keystore. + * @param binaryCommunication should the communication between the CLI and server be binary (CBOR) + * @param apiKey the API key to use for authentication. If null, basic auth will be used if credentials are provided. + * @throws UserException if there is a problem with the information provided by the user + */ + public ConnectionConfiguration buildConnection( + String connectionStringArg, + String keystoreLocation, + boolean binaryCommunication, + String apiKey + ) throws UserException { final URI uri; final String connectionString; Properties properties = new Properties(); @@ -87,7 +105,13 @@ public ConnectionConfiguration buildConnection(String connectionStringArg, Strin properties.put("ssl", "true"); } - if (user != null) { + // API key authentication takes precedence over basic auth + if (apiKey != null) { + if (user != null) { + throw new UserException(ExitCodes.USAGE, "Cannot use both API key and basic authentication"); + } + properties.setProperty(ConnectionConfiguration.AUTH_API_KEY, apiKey); + } else if (user != null) { if (password == null) { password = cliTerminal.readPassword("password: "); } diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilderTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilderTests.java index c004dc869c680..0e45f8dfc1c91 100644 --- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilderTests.java +++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilderTests.java @@ -141,4 +141,29 @@ private ConnectionConfiguration buildConnection(ConnectionBuilder builder, Strin throws UserException { return builder.buildConnection(connectionStringArg, keystoreLocation, randomBoolean()); } + + public void testApiKeyConnection() throws Exception { + CliTerminal testTerminal = mock(CliTerminal.class); + ConnectionBuilder connectionBuilder = new ConnectionBuilder(testTerminal); + String apiKey = "test_api_key_encoded"; + ConnectionConfiguration con = connectionBuilder.buildConnection("http://foobar:9242/", null, randomBoolean(), apiKey); + assertNull(con.authUser()); + assertNull(con.authPass()); + assertEquals(apiKey, con.apiKey()); + assertEquals("http://foobar:9242/", con.connectionString()); + assertEquals(URI.create("http://foobar:9242/"), con.baseUri()); + verifyNoMoreInteractions(testTerminal); + } + + public void testApiKeyAndUserMutuallyExclusive() throws Exception { + CliTerminal testTerminal = mock(CliTerminal.class); + ConnectionBuilder connectionBuilder = new ConnectionBuilder(testTerminal); + String apiKey = "test_api_key_encoded"; + UserException ue = expectThrows( + UserException.class, + () -> connectionBuilder.buildConnection("http://user:pass@foobar:9242/", null, randomBoolean(), apiKey) + ); + assertEquals("Cannot use both API key and basic authentication", ue.getMessage()); + verifyNoMoreInteractions(testTerminal); + } } diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java index 2a6a3fa41ba39..c8a5d048f12c1 100644 --- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java +++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ConnectionConfiguration.java @@ -64,6 +64,7 @@ public class ConnectionConfiguration { public static final String AUTH_USER = "user"; // NB: this is password instead of pass since that's what JDBC DriverManager/tools use public static final String AUTH_PASS = "password"; + public static final String AUTH_API_KEY = "apiKey"; // Default catalog @@ -87,6 +88,7 @@ public class ConnectionConfiguration { PAGE_SIZE, AUTH_USER, AUTH_PASS, + AUTH_API_KEY, CATALOG, ALLOW_PARTIAL_SEARCH_RESULTS, PROJECT_ROUTING @@ -114,6 +116,7 @@ public class ConnectionConfiguration { private final int pageSize; private final String user, pass; + private final String apiKey; private final SslConfig sslConfig; private final ProxyConfig proxyConfig; @@ -152,6 +155,14 @@ public ConnectionConfiguration(URI baseURI, String connectionString, Properties // auth user = settings.getProperty(AUTH_USER); pass = settings.getProperty(AUTH_PASS); + apiKey = settings.getProperty(AUTH_API_KEY); + + // validate that only one authentication method is specified + if (StringUtils.hasText(apiKey) && StringUtils.hasText(user)) { + throw new ClientException( + "Cannot use both API key and basic authentication. Please specify either [" + AUTH_API_KEY + "] or [" + AUTH_USER + "]." + ); + } sslConfig = new SslConfig(settings, baseURI); proxyConfig = new ProxyConfig(settings); @@ -179,6 +190,7 @@ public ConnectionConfiguration( int pageSize, String user, String pass, + String apiKey, SslConfig sslConfig, ProxyConfig proxyConfig, boolean allowPartialSearchResults, @@ -197,6 +209,14 @@ public ConnectionConfiguration( // auth this.user = user; this.pass = pass; + this.apiKey = apiKey; + + // validate that only one authentication method is specified + if (StringUtils.hasText(apiKey) && StringUtils.hasText(user)) { + throw new ClientException( + "Cannot use both API key and basic authentication. Please specify either [" + AUTH_API_KEY + "] or [" + AUTH_USER + "]." + ); + } this.sslConfig = sslConfig; this.proxyConfig = proxyConfig; @@ -307,6 +327,10 @@ public String authPass() { return pass; } + public String apiKey() { + return apiKey; + } + public URI baseUri() { return baseURI; } diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java index 1d35bfe503123..cc0b6271c5912 100644 --- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java +++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java @@ -189,6 +189,7 @@ private boolean head(String path, long timeoutInMs) throws SQLException { cfg.pageSize(), cfg.authUser(), cfg.authPass(), + cfg.apiKey(), cfg.sslConfig(), cfg.proxyConfig(), CoreProtocol.ALLOW_PARTIAL_SEARCH_RESULTS, diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/JreHttpUrlConnection.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/JreHttpUrlConnection.java index 0fe7d15d19b53..7c3250c8d2f91 100644 --- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/JreHttpUrlConnection.java +++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/JreHttpUrlConnection.java @@ -120,7 +120,7 @@ private void setupConnection(ConnectionConfiguration cfg) { // con.setRequestProperty("Accept-Encoding", GZIP); setupSSL(cfg); - setupBasicAuth(cfg); + setupAuth(cfg); } private void setupSSL(ConnectionConfiguration cfg) { @@ -134,8 +134,12 @@ private void setupSSL(ConnectionConfiguration cfg) { } } - private void setupBasicAuth(ConnectionConfiguration cfg) { - if (StringUtils.hasText(cfg.authUser())) { + private void setupAuth(ConnectionConfiguration cfg) { + if (StringUtils.hasText(cfg.apiKey())) { + // API key authentication: Authorization: ApiKey + con.setRequestProperty("Authorization", "ApiKey " + cfg.apiKey()); + } else if (StringUtils.hasText(cfg.authUser())) { + // Basic authentication: Authorization: Basic String basicValue = cfg.authUser() + ":" + cfg.authPass(); String encoded = StringUtils.asUTFString(Base64.getEncoder().encode(StringUtils.toUTF(basicValue))); con.setRequestProperty("Authorization", "Basic " + encoded); diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java index 71c9fb5cee405..85c17fb032994 100644 --- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java +++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/HttpClientRequestTests.java @@ -203,6 +203,51 @@ private void prepareMockResponse() { webServer.enqueue(new Response().setResponseCode(200).addHeader("Content-Type", "application/json").setBody("{\"rows\":[]}")); } + public void testApiKeyAuthentication() throws URISyntaxException { + String url = "http://" + webServer.getHostName() + ":" + webServer.getPort(); + String apiKey = "test_api_key_encoded"; + Properties props = new Properties(); + props.setProperty(ConnectionConfiguration.AUTH_API_KEY, apiKey); + + URI uri = new URI(url); + ConnectionConfiguration conCfg = new ConnectionConfiguration(uri, url, props); + HttpClient httpClient = new HttpClient(conCfg); + + prepareMockResponse(); + try { + httpClient.basicQuery("SELECT 1", 10, false, false); + } catch (SQLException e) { + logger.info("Ignored SQLException", e); + } + assertEquals(1, webServer.requests().size()); + RawRequest recordedRequest = webServer.takeRequest(); + assertEquals("ApiKey " + apiKey, recordedRequest.getHeader("Authorization")); + } + + public void testBasicAuthentication() throws URISyntaxException { + String url = "http://" + webServer.getHostName() + ":" + webServer.getPort(); + String user = "testuser"; + String pass = "testpass"; + Properties props = new Properties(); + props.setProperty(ConnectionConfiguration.AUTH_USER, user); + props.setProperty(ConnectionConfiguration.AUTH_PASS, pass); + + URI uri = new URI(url); + ConnectionConfiguration conCfg = new ConnectionConfiguration(uri, url, props); + HttpClient httpClient = new HttpClient(conCfg); + + prepareMockResponse(); + try { + httpClient.basicQuery("SELECT 1", 10, false, false); + } catch (SQLException e) { + logger.info("Ignored SQLException", e); + } + assertEquals(1, webServer.requests().size()); + RawRequest recordedRequest = webServer.takeRequest(); + String expectedAuth = "Basic " + java.util.Base64.getEncoder().encodeToString((user + ":" + pass).getBytes(StandardCharsets.UTF_8)); + assertEquals(expectedAuth, recordedRequest.getHeader("Authorization")); + } + @SuppressForbidden(reason = "use http server") private static class RawRequestMockWebServer implements Closeable { private HttpServer server;