From 1e6b323e4e35a3418901984607616554db87f914 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 27 Mar 2019 20:09:30 +0200 Subject: [PATCH 01/50] Rename drudgery --- .../index/RestrictedIndicesNames.java | 13 ++- .../authz/store/ReservedRolesStoreTests.java | 22 ++-- .../xpack/security/Security.java | 16 +-- .../xpack/security/authc/ApiKeyService.java | 12 +-- .../security/authc/ExpiredApiKeysRemover.java | 4 +- .../security/authc/ExpiredTokenRemover.java | 4 +- .../xpack/security/authc/TokenService.java | 69 ++++++------ .../authc/esnative/NativeUsersStore.java | 26 ++--- .../mapper/NativeRoleMappingStore.java | 14 +-- .../authz/store/NativePrivilegeStore.java | 10 +- .../authz/store/NativeRolesStore.java | 18 ++-- .../support/SecurityIndexManager.java | 38 ++++--- .../integration/ClearRolesCacheTests.java | 4 +- .../test/NativeRealmIntegTestCase.java | 2 +- .../test/SecurityIntegTestCase.java | 8 +- .../xpack/security/SecurityTests.java | 16 +-- .../security/authc/ApiKeyIntegTests.java | 9 +- .../authc/AuthenticationServiceTests.java | 2 +- .../security/authc/TokenAuthIntegTests.java | 15 ++- .../esnative/ESNativeMigrateToolTests.java | 5 +- .../authc/esnative/NativeRealmIntegTests.java | 34 +++--- .../authc/esnative/NativeRealmTests.java | 2 +- .../authc/esnative/NativeUsersStoreTests.java | 8 +- .../mapper/NativeRoleMappingStoreTests.java | 2 +- .../authz/AuthorizationServiceTests.java | 101 +++++++++--------- .../authz/AuthorizedIndicesTests.java | 27 +++-- .../authz/IndicesAndAliasesResolverTests.java | 19 ++-- .../authz/SnapshotUserRoleIntegTests.java | 14 +-- .../accesscontrol/IndicesPermissionTests.java | 14 +-- .../authz/store/CompositeRolesStoreTests.java | 2 +- .../store/NativePrivilegeStoreTests.java | 17 +-- .../authz/store/NativeRolesStoreTests.java | 8 +- .../support/SecurityIndexManagerTests.java | 74 +++++++------ .../security/test/SecurityTestUtils.java | 4 +- .../xpack/security/user/XPackUserTests.java | 3 - .../xpack/restart/FullClusterRestartIT.java | 3 +- .../ldap/AbstractAdLdapRealmTestCase.java | 2 +- 37 files changed, 329 insertions(+), 312 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java index 439168350fc7e..80c17c484739c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java @@ -14,12 +14,15 @@ import java.util.Set; public final class RestrictedIndicesNames { - public static final String INTERNAL_SECURITY_INDEX_6 = ".security-6"; - public static final String INTERNAL_SECURITY_INDEX_7 = ".security-7"; - public static final String SECURITY_INDEX_NAME = ".security"; + public static final String INTERNAL_SECURITY_MAIN_INDEX_6 = ".security-6"; + public static final String INTERNAL_SECURITY_MAIN_INDEX_7 = ".security-7"; + public static final String SECURITY_MAIN_ALIAS = ".security"; - public static final Set RESTRICTED_NAMES = Collections.unmodifiableSet( - Sets.newHashSet(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX_6, INTERNAL_SECURITY_INDEX_7)); + public static final String INTERNAL_SECURITY_TOKENS_INDEX_7 = ".security-tokens-7"; + public static final String SECURITY_TOKENS_ALIAS = ".security-tokens"; + + public static final Set RESTRICTED_NAMES = Collections.unmodifiableSet(Sets.newHashSet(SECURITY_MAIN_ALIAS, + INTERNAL_SECURITY_MAIN_INDEX_6, INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_SECURITY_TOKENS_INDEX_7, SECURITY_TOKENS_ALIAS)); public static final Automaton NAMES_AUTOMATON = Automatons.patterns(RESTRICTED_NAMES); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 9d970cca55119..757d766eb4a42 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -589,14 +589,14 @@ public void testRemoteMonitoringCollectorRole() { private void assertMonitoringOnRestrictedIndices(Role role) { final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); - final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); final MetaData metaData = new MetaData.Builder() .put(new IndexMetaData.Builder(internalSecurityIndex) .settings(indexSettings) .numberOfShards(1) .numberOfReplicas(0) - .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build()) + .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build()) .build(), true) .build(); final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); @@ -604,10 +604,10 @@ private void assertMonitoringOnRestrictedIndices(Role role) { GetSettingsAction.NAME, IndicesShardStoresAction.NAME, UpgradeStatusAction.NAME, RecoveryAction.NAME); for (final String indexMonitoringActionName : indexMonitoringActionNamesList) { final Map authzMap = role.indices().authorize(indexMonitoringActionName, - Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_INDEX_NAME), + Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_MAIN_ALIAS), metaData.getAliasAndIndexLookup(), fieldPermissionsCache); assertThat(authzMap.get(internalSecurityIndex).isGranted(), is(true)); - assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true)); + assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).isGranted(), is(true)); } } @@ -701,8 +701,8 @@ public void testSuperuserRole() { assertThat(superuserRole.cluster().check("internal:admin/foo", request), is(false)); final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); - final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); final MetaData metaData = new MetaData.Builder() .put(new IndexMetaData.Builder("a1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) .put(new IndexMetaData.Builder("a2").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) @@ -719,7 +719,7 @@ public void testSuperuserRole() { .settings(indexSettings) .numberOfShards(1) .numberOfReplicas(0) - .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build()) + .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build()) .build(), true) .build(); @@ -741,8 +741,8 @@ public void testSuperuserRole() { assertThat(authzMap.get("aaaaaa").isGranted(), is(true)); assertThat(authzMap.get("b").isGranted(), is(true)); authzMap = superuserRole.indices().authorize(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME), - Sets.newHashSet(RestrictedIndicesNames.SECURITY_INDEX_NAME), lookup, fieldPermissionsCache); - assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true)); + Sets.newHashSet(RestrictedIndicesNames.SECURITY_MAIN_ALIAS), lookup, fieldPermissionsCache); + assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).isGranted(), is(true)); assertThat(authzMap.get(internalSecurityIndex).isGranted(), is(true)); assertTrue(superuserRole.indices().check(SearchAction.NAME)); assertFalse(superuserRole.indices().check("unknown")); @@ -750,7 +750,7 @@ public void testSuperuserRole() { assertThat(superuserRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(true)); assertThat(superuserRole.indices().allowedIndicesMatcher(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME)) - .test(RestrictedIndicesNames.SECURITY_INDEX_NAME), is(true)); + .test(RestrictedIndicesNames.SECURITY_MAIN_ALIAS), is(true)); assertThat(superuserRole.indices().allowedIndicesMatcher(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME)) .test(internalSecurityIndex), is(true)); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index fdfc6c9a59498..dbb0053637857 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -248,9 +248,9 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE; public class Security extends Plugin implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin, MapperPlugin, ExtensiblePlugin { @@ -396,7 +396,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(auditTrailService); this.auditTrailService.set(auditTrailService); - securityIndex.set(SecurityIndexManager.buildSecurityIndexManager(client, clusterService)); + securityIndex.set(SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService)); final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService); this.tokenService.set(tokenService); @@ -922,7 +922,7 @@ public List> getExecutorBuilders(final Settings settings) { public UnaryOperator> getIndexTemplateMetaDataUpgrader() { return templates -> { // .security index is not managed by using templates anymore - templates.remove(SECURITY_TEMPLATE_NAME); + templates.remove(SECURITY_MAIN_TEMPLATE); templates.remove("security_audit_log"); return templates; }; @@ -989,9 +989,9 @@ static final class ValidateUpgradedSecurityIndex implements BiConsumerwrap(response -> { @@ -727,7 +727,7 @@ private void findApiKeys(final BoolQueryBuilder boolQuery, boolean filterOutInva expiredQuery.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("expiration_time"))); boolQuery.filter(expiredQuery); } - final SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME) + final SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setQuery(boolQuery) .setVersion(false) @@ -801,7 +801,7 @@ private void indexInvalidation(Collection apiKeyIds, ActionListener listener) securityIndex.checkIndexVersionThenExecute( ex -> listener.onFailure(traceLog("prepare security index", userTokenId, ex)), () -> { - final GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, + final GetRequest getRequest = client.prepareGet(securityIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userTokenId)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("decode token", userTokenId, ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, @@ -461,30 +460,6 @@ void decodeToken(String token, ActionListener listener) { } } - private void getKeyAsync(BytesKey decodedSalt, KeyAndCache keyAndCache, ActionListener listener) { - final SecretKey decodeKey = keyAndCache.getKey(decodedSalt); - if (decodeKey != null) { - listener.onResponse(decodeKey); - } else { - /* As a measure of protected against DOS, we can pass requests requiring a key - * computation off to a single thread executor. For normal usage, the initial - * request(s) that require a key computation will be delayed and there will be - * some additional latency. - */ - client.threadPool().executor(THREAD_POOL_NAME) - .submit(new KeyComputingRunnable(decodedSalt, keyAndCache, listener)); - } - } - - private static String decryptTokenId(byte[] encryptedTokenId, Cipher cipher, Version version) throws IOException { - try (ByteArrayInputStream bais = new ByteArrayInputStream(encryptedTokenId); - CipherInputStream cis = new CipherInputStream(bais, cipher); - StreamInput decryptedInput = new InputStreamStreamInput(cis)) { - decryptedInput.setVersion(version); - return decryptedInput.readString(); - } - } - /** * This method performs the steps necessary to invalidate an access token so that it may no longer be * used. The process of invalidation involves performing an update to the token document and setting @@ -627,7 +602,7 @@ private void indexInvalidation(Collection tokenIds, Iterator BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); for (String tokenId : tokenIds) { UpdateRequest request = client - .prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getTokenDocumentId(tokenId)) + .prepareUpdate(securityIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(tokenId)) .setDoc(srcPrefix, Collections.singletonMap("invalidated", true)) .setFetchSource(srcPrefix, null) .request(); @@ -746,7 +721,7 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator logger.debug("security index is not available to find token from refresh token, retrying"); maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); } else { - final SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME) + final SearchRequest request = client.prepareSearch(securityIndex.aliasName()) .setQuery(QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) .filter(QueryBuilders.termQuery("refresh_token.token", refreshToken))) @@ -863,7 +838,7 @@ public void onFailure(Exception e) { updateMap.put("superseded_by", getTokenDocumentId(newUserTokenId)); assert seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; assert primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM : "expected an assigned primary term"; - final UpdateRequestBuilder updateRequest = client.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, tokenDocId) + final UpdateRequestBuilder updateRequest = client.prepareUpdate(securityIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) .setDoc("refresh_token", updateMap) .setFetchSource(true) .setRefreshPolicy(RefreshPolicy.IMMEDIATE) @@ -943,7 +918,7 @@ public void onFailure(Exception e) { } private void getTokenDocAsync(String tokenDocId, ActionListener listener) { - final GetRequest getRequest = client.prepareGet(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, tokenDocId).request(); + final GetRequest getRequest = client.prepareGet(securityIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, listener, client::get); } @@ -1084,7 +1059,7 @@ public void findActiveTokensForRealm(String realmName, @Nullable Predicate listener.onResponse(null); } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { - final GetRequest getRequest = client.prepareGet(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getTokenDocumentId(userToken)) - .request(); + final GetRequest getRequest = client + .prepareGet(securityIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userToken)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("check token state", userToken.getId(), ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { @@ -1344,6 +1319,30 @@ Cipher getEncryptionCipher(byte[] iv, KeyAndCache keyAndCache, Version version) return cipher; } + private void getKeyAsync(BytesKey decodedSalt, KeyAndCache keyAndCache, ActionListener listener) { + final SecretKey decodeKey = keyAndCache.getKey(decodedSalt); + if (decodeKey != null) { + listener.onResponse(decodeKey); + } else { + /* As a measure of protected against DOS, we can pass requests requiring a key + * computation off to a single thread executor. For normal usage, the initial + * request(s) that require a key computation will be delayed and there will be + * some additional latency. + */ + client.threadPool().executor(THREAD_POOL_NAME) + .submit(new KeyComputingRunnable(decodedSalt, keyAndCache, listener)); + } + } + + private static String decryptTokenId(byte[] encryptedTokenId, Cipher cipher, Version version) throws IOException { + try (ByteArrayInputStream bais = new ByteArrayInputStream(encryptedTokenId); + CipherInputStream cis = new CipherInputStream(bais, cipher); + StreamInput decryptedInput = new InputStreamStreamInput(cis)) { + decryptedInput.setVersion(version); + return decryptedInput.readString(); + } + } + private Cipher getDecryptionCipher(byte[] iv, SecretKey key, Version version, BytesKey salt) throws GeneralSecurityException { Cipher cipher = Cipher.getInstance(ENCRYPTION_CIPHER); cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(128, iv), secureRandom); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 3ae7373d2e2e3..e5a213331431f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -67,7 +67,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; /** * NativeUsersStore is a store for users that reads from an Elasticsearch index. This store is responsible for fetching the full @@ -147,7 +147,7 @@ public void getUsers(String[] userNames, final ActionListener> } final Supplier supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setQuery(query) .setSize(1000) @@ -172,7 +172,7 @@ void getUserCount(final ActionListener listener) { } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareSearch(SECURITY_INDEX_NAME) + client.prepareSearch(SECURITY_MAIN_ALIAS) .setQuery(QueryBuilders.termQuery(Fields.TYPE.getPreferredName(), USER_DOC_TYPE)) .setSize(0) .setTrackTotalHits(true) @@ -206,7 +206,7 @@ private void getUserAndPassword(final String user, final ActionListener executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, user)).request(), + client.prepareGet(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, user)).request(), new ActionListener() { @Override public void onResponse(GetResponse response) { @@ -246,7 +246,7 @@ public void changePassword(final ChangePasswordRequest request, final ActionList securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(docType, username)) + client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(docType, username)) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.PASSWORD.getPreferredName(), String.valueOf(request.passwordHash())) .setRefreshPolicy(request.getRefreshPolicy()).request(), @@ -284,7 +284,7 @@ public void onFailure(Exception e) { private void createReservedUser(String username, char[] passwordHash, RefreshPolicy refresh, ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareIndex(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(RESERVED_USER_TYPE, username)) + client.prepareIndex(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(RESERVED_USER_TYPE, username)) .setSource(Fields.PASSWORD.getPreferredName(), String.valueOf(passwordHash), Fields.ENABLED.getPreferredName(), true, Fields.TYPE.getPreferredName(), RESERVED_USER_TYPE) .setRefreshPolicy(refresh).request(), @@ -324,7 +324,7 @@ private void updateUserWithoutPassword(final PutUserRequest putUserRequest, fina // We must have an existing document securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, putUserRequest.username())) + client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, putUserRequest.username())) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.USERNAME.getPreferredName(), putUserRequest.username(), Fields.ROLES.getPreferredName(), putUserRequest.roles(), @@ -368,7 +368,7 @@ private void indexUser(final PutUserRequest putUserRequest, final ActionListener assert putUserRequest.passwordHash() != null; securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareIndex(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, putUserRequest.username())) + client.prepareIndex(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, putUserRequest.username())) .setSource(Fields.USERNAME.getPreferredName(), putUserRequest.username(), Fields.PASSWORD.getPreferredName(), String.valueOf(putUserRequest.passwordHash()), Fields.ROLES.getPreferredName(), putUserRequest.roles(), @@ -411,7 +411,7 @@ private void setRegularUserEnabled(final String username, final boolean enabled, final ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, username)) + client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, username)) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.ENABLED.getPreferredName(), enabled) .setRefreshPolicy(refreshPolicy) .request(), @@ -445,7 +445,7 @@ private void setReservedUserEnabled(final String username, final boolean enabled boolean clearCache, final ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(RESERVED_USER_TYPE, username)) + client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(RESERVED_USER_TYPE, username)) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.ENABLED.getPreferredName(), enabled) .setUpsert(XContentType.JSON, Fields.PASSWORD.getPreferredName(), "", @@ -480,7 +480,7 @@ public void deleteUser(final DeleteUserRequest deleteUserRequest, final ActionLi } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { DeleteRequest request = client - .prepareDelete(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, deleteUserRequest.username())) + .prepareDelete(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(USER_DOC_TYPE, deleteUserRequest.username())) .request(); request.setRefreshPolicy(deleteUserRequest.getRefreshPolicy()); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, @@ -527,7 +527,7 @@ void getReservedUserInfo(String username, ActionListener liste } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForUser(RESERVED_USER_TYPE, username)) + client.prepareGet(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForUser(RESERVED_USER_TYPE, username)) .request(), new ActionListener() { @Override @@ -572,7 +572,7 @@ void getAllReservedUserInfo(ActionListener> listen } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareSearch(SECURITY_INDEX_NAME) + client.prepareSearch(SECURITY_MAIN_ALIAS) .setTrackTotalHits(true) .setQuery(QueryBuilders.termQuery(Fields.TYPE.getPreferredName(), RESERVED_USER_TYPE)) .setFetchSource(true).request(), diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index cbb352e67ab39..08bbcc1046f4d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -61,13 +61,13 @@ import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; /** * This store reads + writes {@link ExpressionRoleMapping role mappings} in an Elasticsearch - * {@link SecurityIndexManager#SECURITY_INDEX_NAME index}. + * {@link SecurityIndexManager#SECURITY_MAIN_ALIAS_NAME index}. *
* The store is responsible for all read and write operations as well as * {@link #resolveRoles(UserData, ActionListener) resolving roles}. @@ -130,7 +130,7 @@ void loadMappings(ActionListener> listener) { final QueryBuilder query = QueryBuilders.termQuery(DOC_TYPE_FIELD, DOC_TYPE_ROLE_MAPPING); final Supplier supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setQuery(query) .setSize(1000) @@ -142,7 +142,7 @@ void loadMappings(ActionListener> listener) { listener.onResponse(mappings.stream().filter(Objects::nonNull).collect(Collectors.toList())), ex -> { logger.error(new ParameterizedMessage("failed to load role mappings from index [{}] skipping all mappings.", - SECURITY_INDEX_NAME), ex); + SECURITY_MAIN_ALIAS), ex); listener.onResponse(Collections.emptyList()); })), doc -> buildMapping(getNameFromId(doc.getId()), doc.getSourceRef())); @@ -201,7 +201,7 @@ private void innerPutMapping(PutRoleMappingRequest request, ActionListener { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareDelete(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForName(request.getName())) + client.prepareDelete(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForName(request.getName())) .setRefreshPolicy(request.getRefreshPolicy()) .request(), new ActionListener() { @@ -285,7 +285,7 @@ private void getMappings(ActionListener> listener) { logger.info("The security index is not yet available - no role mappings can be loaded"); if (logger.isDebugEnabled()) { logger.debug("Security Index [{}] [exists: {}] [available: {}] [mapping up to date: {}]", - SECURITY_INDEX_NAME, + SECURITY_MAIN_ALIAS, securityIndex.indexExists(), securityIndex.isAvailable(), securityIndex.isMappingUpToDate() diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java index 09c89752f8314..a5962894255a8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java @@ -62,7 +62,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; import static org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor.DOC_TYPE_VALUE; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; /** * {@code NativePrivilegeStore} is a store that reads/writes {@link ApplicationPrivilegeDescriptor} objects, @@ -123,7 +123,7 @@ public void getPrivileges(Collection applications, Collection na } final Supplier supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setQuery(query) .setSize(1000) @@ -152,7 +152,7 @@ void getPrivilege(String application, String name, ActionListener executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, toDocId(application, name)) + client.prepareGet(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, toDocId(application, name)) .request(), new ActionListener() { @Override @@ -204,7 +204,7 @@ private void innerPutPrivilege(ApplicationPrivilegeDescriptor privilege, WriteRe final String name = privilege.getName(); final XContentBuilder xContentBuilder = privilege.toXContent(jsonBuilder(), true); ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareIndex(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, toDocId(privilege.getApplication(), name)) + client.prepareIndex(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, toDocId(privilege.getApplication(), name)) .setSource(xContentBuilder) .setRefreshPolicy(refreshPolicy) .request(), listener, client::index); @@ -235,7 +235,7 @@ public void deletePrivileges(String application, Collection names, Write }, listener::onFailure), names.size()); for (String name : names) { ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareDelete(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, toDocId(application, name)) + client.prepareDelete(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, toDocId(application, name)) .setRefreshPolicy(refreshPolicy) .request(), groupListener, client::delete); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index b82bf7f3c7fc2..157bf145d05d0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -71,7 +71,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; import static org.elasticsearch.xpack.core.security.SecurityField.setting; import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ROLE_TYPE; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; /** * NativeRolesStore is a {@code RolesStore} that, instead of reading from a @@ -125,7 +125,7 @@ public void getRoleDescriptors(Set names, final ActionListener supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS) .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) .setQuery(query) .setSize(1000) @@ -143,7 +143,7 @@ public void getRoleDescriptors(Set names, final ActionListener { final String[] roleIds = names.stream().map(NativeRolesStore::getIdForRole).toArray(String[]::new); - MultiGetRequest multiGetRequest = client.prepareMultiGet().add(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, roleIds).request(); + MultiGetRequest multiGetRequest = client.prepareMultiGet().add(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, roleIds).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, multiGetRequest, ActionListener.wrap(mGetResponse -> { final MultiGetItemResponse[] responses = mGetResponse.getResponses(); @@ -180,7 +180,7 @@ public void deleteRole(final DeleteRoleRequest deleteRoleRequest, final ActionLi } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { DeleteRequest request = client - .prepareDelete(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForRole(deleteRoleRequest.name())).request(); + .prepareDelete(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForRole(deleteRoleRequest.name())).request(); request.setRefreshPolicy(deleteRoleRequest.getRefreshPolicy()); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, new ActionListener() { @@ -220,7 +220,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final listener.onFailure(e); return; } - final IndexRequest indexRequest = client.prepareIndex(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForRole(role.getName())) + final IndexRequest indexRequest = client.prepareIndex(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForRole(role.getName())) .setSource(xContentBuilder) .setRefreshPolicy(request.getRefreshPolicy()) .request(); @@ -254,11 +254,11 @@ public void usageStats(ActionListener> listener) { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareMultiSearch() - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SECURITY_MAIN_ALIAS) .setQuery(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .setTrackTotalHits(true) .setSize(0)) - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SECURITY_MAIN_ALIAS) .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .must(QueryBuilders.boolQuery() @@ -269,7 +269,7 @@ public void usageStats(ActionListener> listener) { .setTrackTotalHits(true) .setSize(0) .setTerminateAfter(1)) - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SECURITY_MAIN_ALIAS) .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .filter(existsQuery("indices.query"))) @@ -341,7 +341,7 @@ public void onFailure(Exception e) { private void executeGetRoleRequest(String role, ActionListener listener) { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, getIdForRole(role)).request(), + client.prepareGet(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, getIdForRole(role)).request(), listener, client::get)); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 9a316a9ace4c6..468a9a535c5f9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -73,16 +73,17 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; /** - * Manages the lifecycle of a single index, mapping and and data upgrades/migrations. + * Manages the lifecycle, mapping and data upgrades/migrations of one of `.security-7` or `.security-tokens-7` indices. */ public class SecurityIndexManager implements ClusterStateListener { - public static final String INTERNAL_SECURITY_INDEX = RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7; - public static final int INTERNAL_INDEX_FORMAT = 6; + public static final int INTERNAL_MAIN_INDEX_FORMAT = 6; + public static final int INTERNAL_TOKENS_INDEX_FORMAT = 7; + public static final String SECURITY_MAIN_TEMPLATE = "security-index-template"; + public static final String SECURITY_TOKENS_TEMPLATE = "security-tokens-index-template"; public static final String SECURITY_VERSION_STRING = "security-version"; public static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); - public static final String SECURITY_TEMPLATE_NAME = "security-index-template"; - public static final String SECURITY_INDEX_NAME = ".security"; + private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class); private final String aliasName; @@ -95,19 +96,26 @@ public class SecurityIndexManager implements ClusterStateListener { private volatile State indexState; - public static SecurityIndexManager buildSecurityIndexManager(Client client, ClusterService clusterService) { - return new SecurityIndexManager(client, SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX, INTERNAL_INDEX_FORMAT, - SecurityIndexManager::readSecurityTemplateAsBytes, clusterService); + public static SecurityIndexManager buildSecurityMainIndexManager(Client client, ClusterService clusterService) { + return new SecurityIndexManager(client, clusterService, RestrictedIndicesNames.SECURITY_MAIN_ALIAS, + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_MAIN_INDEX_FORMAT, + () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_MAIN_TEMPLATE)); } - private SecurityIndexManager(Client client, String aliasName, String internalIndexName, int internalIndexFormat, - Supplier mappingSourceSupplier, ClusterService clusterService) { + public static SecurityIndexManager buildSecurityTokensIndexManager(Client client, ClusterService clusterService) { + return new SecurityIndexManager(client, clusterService, RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, + RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, INTERNAL_TOKENS_INDEX_FORMAT, + () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_TOKENS_TEMPLATE)); + } + + private SecurityIndexManager(Client client, ClusterService clusterService, String aliasName, String internalIndexName, + int internalIndexFormat, Supplier mappingSourceSupplier) { this(client, aliasName, internalIndexName, internalIndexFormat, mappingSourceSupplier, State.UNRECOVERED_STATE); clusterService.addListener(this); } private SecurityIndexManager(Client client, String aliasName, String internalIndexName, int internalIndexFormat, - Supplier mappingSourceSupplier, State indexState) { + Supplier mappingSourceSupplier, State indexState) { this.aliasName = aliasName; this.internalIndexName = internalIndexName; this.internalIndexFormat = internalIndexFormat; @@ -126,6 +134,10 @@ public boolean checkMappingVersion(Predicate requiredVersion) { return currentIndexState.mappingVersion == null || requiredVersion.test(currentIndexState.mappingVersion); } + public String aliasName() { + return aliasName; + } + public boolean indexExists() { return this.indexState.indexExists; } @@ -396,8 +408,8 @@ public static boolean isIndexDeleted(State previousState, State currentState) { return previousState.indexStatus != null && currentState.indexStatus == null; } - private static byte[] readSecurityTemplateAsBytes() { - return TemplateUtils.loadTemplate("/" + SECURITY_TEMPLATE_NAME + ".json", Version.CURRENT.toString(), + private static byte[] readTemplateAsBytes(String templateName) { + return TemplateUtils.loadTemplate("/" + templateName + ".json", Version.CURRENT.toString(), SecurityIndexManager.TEMPLATE_VERSION_PATTERN).getBytes(StandardCharsets.UTF_8); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java index c7f5123c6a1b7..6d7eacfe26cfa 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java @@ -13,8 +13,8 @@ import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse; import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult; import org.elasticsearch.xpack.core.security.client.SecurityClient; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.Before; import org.junit.BeforeClass; @@ -56,7 +56,7 @@ public void setupForTests() { logger.debug("--> created role [{}]", role); } - ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); + ensureGreen(RestrictedIndicesNames.SECURITY_MAIN_ALIAS); final Set rolesSet = new HashSet<>(Arrays.asList(roles)); // warm up the caches on every node diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index 63f5ace535229..e08ea8b6022f2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -59,7 +59,7 @@ protected boolean addMockHttpTransport() { @Override public Set excludeTemplates() { Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityIndexManager.SECURITY_TEMPLATE_NAME); // don't remove the security index template + templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE); // don't remove the security index template return templates; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index c3e3bddf10e97..bd9d58e6ea549 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -67,7 +67,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsCollectionContaining.hasItem; @@ -491,7 +491,7 @@ public void assertSecurityIndexActive(TestCluster testCluster) throws Exception XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint().startObject(); assertTrue("security index mapping not sufficient to read:\n" + Strings.toString(clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject()), - SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, logger, + SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_MAIN_ALIAS, clusterState, logger, Version.CURRENT.minimumIndexCompatibilityVersion()::onOrBefore)); Index securityIndex = resolveSecurityIndex(clusterState.metaData()); if (securityIndex != null) { @@ -509,7 +509,7 @@ protected void deleteSecurityIndex() { UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING))); GetIndexRequest getIndexRequest = new GetIndexRequest(); - getIndexRequest.indices(SECURITY_INDEX_NAME); + getIndexRequest.indices(SECURITY_MAIN_ALIAS); getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); GetIndexResponse getIndexResponse = client.admin().indices().getIndex(getIndexRequest).actionGet(); if (getIndexResponse.getIndices().length > 0) { @@ -520,7 +520,7 @@ protected void deleteSecurityIndex() { } private static Index resolveSecurityIndex(MetaData metaData) { - final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(SECURITY_INDEX_NAME); + final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(SECURITY_MAIN_ALIAS); if (aliasOrIndex != null) { return aliasOrIndex.getIndices().get(0).getIndex(); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 35180ab8f31d0..fa6f9025186ba 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -66,8 +66,8 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.discovery.DiscoveryModule.ZEN2_DISCOVERY_TYPE; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -308,8 +308,8 @@ public void testIndexJoinValidator_Old_And_Rolling() throws Exception { BiConsumer joinValidator = security.getJoinValidator(); assertNotNull(joinValidator); DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); - IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_INDEX_NAME) - .settings(settings(Version.V_6_1_0).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_INDEX_FORMAT - 1)) + IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_MAIN_ALIAS) + .settings(settings(Version.V_6_1_0).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_MAIN_INDEX_FORMAT - 1)) .numberOfShards(1).numberOfReplicas(0) .build(); DiscoveryNode existingOtherNode = new DiscoveryNode("bar", buildNewFakeTransportAddress(), Version.V_6_1_0); @@ -328,8 +328,8 @@ public void testIndexJoinValidator_FullyCurrentCluster() throws Exception { BiConsumer joinValidator = security.getJoinValidator(); assertNotNull(joinValidator); DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); - int indexFormat = randomBoolean() ? INTERNAL_INDEX_FORMAT : INTERNAL_INDEX_FORMAT - 1; - IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_INDEX_NAME) + int indexFormat = randomBoolean() ? INTERNAL_MAIN_INDEX_FORMAT : INTERNAL_MAIN_INDEX_FORMAT - 1; + IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_MAIN_ALIAS) .settings(settings(Version.V_6_1_0).put(INDEX_FORMAT_SETTING.getKey(), indexFormat)) .numberOfShards(1).numberOfReplicas(0) .build(); @@ -347,8 +347,8 @@ public void testIndexUpgradeValidatorWithUpToDateIndex() throws Exception { assertNotNull(joinValidator); Version version = randomBoolean() ? Version.CURRENT : Version.V_6_1_0; DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); - IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_INDEX_NAME) - .settings(settings(version).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_INDEX_FORMAT)) + IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_MAIN_ALIAS) + .settings(settings(version).put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_MAIN_INDEX_FORMAT)) .numberOfShards(1).numberOfReplicas(0) .build(); DiscoveryNode existingOtherNode = new DiscoveryNode("bar", buildNewFakeTransportAddress(), version); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 256bf6d9df532..071136d8608bf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -53,7 +53,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -336,7 +336,7 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore() Instant dayBefore = created.minus(1L, ChronoUnit.DAYS); assertTrue(Instant.now().isAfter(dayBefore)); UpdateResponse expirationDateUpdatedResponse = client - .prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, createdApiKeys.get(0).getId()) + .prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, createdApiKeys.get(0).getId()) .setDoc("expiration_time", dayBefore.toEpochMilli()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -346,7 +346,7 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore() // hack doc to modify the expiration time to the week before Instant weekBefore = created.minus(8L, ChronoUnit.DAYS); assertTrue(Instant.now().isAfter(weekBefore)); - expirationDateUpdatedResponse = client.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, createdApiKeys.get(1).getId()) + expirationDateUpdatedResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, createdApiKeys.get(1).getId()) .setDoc("expiration_time", weekBefore.toEpochMilli()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -390,8 +390,7 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore() private void refreshSecurityIndex() throws Exception { assertBusy(() -> { - final RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(SecurityIndexManager.SECURITY_INDEX_NAME) - .get(); + final RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(SECURITY_MAIN_ALIAS).get(); assertThat(refreshResponse.getFailedShards(), is(0)); }); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 63e8578e8c2fc..e05dd74d1275b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -154,7 +154,7 @@ public class AuthenticationServiceTests extends ESTestCase { @SuppressForbidden(reason = "Allow accessing localhost") public void init() throws Exception { concreteSecurityIndexName = randomFrom( - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); token = mock(AuthenticationToken.class); when(token.principal()).thenReturn(randomAlphaOfLength(5)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index 6bcdfc94e9957..c52896660a650 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.xpack.core.security.authc.TokenMetaData; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.Before; @@ -55,7 +54,7 @@ import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.equalTo; @TestLogging("org.elasticsearch.xpack.security.authz.store.FileRolesStore:DEBUG") @@ -161,7 +160,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); AtomicReference docId = new AtomicReference<>(); assertBusy(() -> { - SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + SearchResponse searchResponse = client.prepareSearch(SECURITY_MAIN_ALIAS) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.termQuery("doc_type", "token"))) .setSize(1) @@ -174,7 +173,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { // hack doc to modify the creation time to the day before Instant yesterday = created.minus(36L, ChronoUnit.HOURS); assertTrue(Instant.now().isAfter(yesterday)); - client.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, docId.get()) + client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, docId.get()) .setDoc("creation_time", yesterday.toEpochMilli()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -192,8 +191,8 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { assertEquals("token malformed", e.getMessage()); } } - client.admin().indices().prepareRefresh(SecurityIndexManager.SECURITY_INDEX_NAME).get(); - SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + client.admin().indices().prepareRefresh(SECURITY_MAIN_ALIAS).get(); + SearchResponse searchResponse = client.prepareSearch(SECURITY_MAIN_ALIAS) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.termQuery("doc_type", "token"))) .setTerminateAfter(1) @@ -358,7 +357,7 @@ public void testRefreshingMultipleTimesFails() throws Exception { // We now have two documents, the original(now refreshed) token doc and the new one with the new access doc AtomicReference docId = new AtomicReference<>(); assertBusy(() -> { - SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + SearchResponse searchResponse = client.prepareSearch(SECURITY_MAIN_ALIAS) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery("doc_type", "token")) @@ -374,7 +373,7 @@ public void testRefreshingMultipleTimesFails() throws Exception { Instant refreshed = Instant.now(); Instant aWhileAgo = refreshed.minus(50L, ChronoUnit.SECONDS); assertTrue(Instant.now().isAfter(aWhileAgo)); - UpdateResponse updateResponse = client.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, docId.get()) + UpdateResponse updateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, docId.get()) .setDoc("refresh_token", Collections.singletonMap("refresh_time", aWhileAgo.toEpochMilli())) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .setFetchSource("refresh_token", Strings.EMPTY_STRING) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java index 650ccc55e41b3..9cd95777055f4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.test.NativeRealmIntegTestCase; import org.elasticsearch.common.CharArrays; import org.elasticsearch.xpack.core.security.client.SecurityClient; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.BeforeClass; @@ -85,7 +86,7 @@ public void testRetrieveUsers() throws Exception { addedUsers.add(uname); } logger.error("--> waiting for .security index"); - ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); + ensureGreen(RestrictedIndicesNames.SECURITY_MAIN_ALIAS); MockTerminal t = new MockTerminal(); String username = nodeClientUsername(); @@ -136,7 +137,7 @@ public void testRetrieveRoles() throws Exception { addedRoles.add(rname); } logger.error("--> waiting for .security index"); - ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); + ensureGreen(RestrictedIndicesNames.SECURITY_MAIN_ALIAS); MockTerminal t = new MockTerminal(); String username = nodeClientUsername(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java index d547fe5a83960..0c109b981ef7c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.client.SecurityClient; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; @@ -67,8 +68,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_SECURITY_INDEX; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.contains; @@ -146,7 +147,7 @@ public void testAddAndGetUser() throws Exception { logger.error("--> creating user"); c.preparePutUser("joe", "s3kirt".toCharArray(), hasher, "role1", "user").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_MAIN_ALIAS); logger.info("--> retrieving user"); GetUsersResponse resp = c.prepareGetUsers("joe").get(); assertTrue("user should exist", resp.hasUsers()); @@ -201,7 +202,7 @@ public void testAddAndGetRole() throws Exception { .metadata(metadata) .get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_MAIN_ALIAS); logger.info("--> retrieving role"); GetRolesResponse resp = c.prepareGetRoles().names("test_role").get(); assertTrue("role should exist", resp.hasRoles()); @@ -252,7 +253,7 @@ public void testAddUserAndRoleThenAuth() throws Exception { logger.error("--> creating user"); c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_MAIN_ALIAS); logger.info("--> retrieving user"); GetUsersResponse resp = c.prepareGetUsers("joe").get(); assertTrue("user should exist", resp.hasUsers()); @@ -273,7 +274,7 @@ public void testUpdatingUserAndAuthentication() throws Exception { logger.error("--> creating user"); c.preparePutUser("joe", "s3krit".toCharArray(), hasher, SecuritySettingsSource.TEST_ROLE).get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_MAIN_ALIAS); logger.info("--> retrieving user"); GetUsersResponse resp = c.prepareGetUsers("joe").get(); assertTrue("user should exist", resp.hasUsers()); @@ -309,7 +310,7 @@ public void testCreateDeleteAuthenticate() { c.preparePutUser("joe", "s3krit".toCharArray(), hasher, SecuritySettingsSource.TEST_ROLE).get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_MAIN_ALIAS); logger.info("--> retrieving user"); GetUsersResponse resp = c.prepareGetUsers("joe").get(); assertTrue("user should exist", resp.hasUsers()); @@ -347,7 +348,7 @@ public void testCreateAndUpdateRole() { logger.error("--> creating user"); c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_MAIN_ALIAS); if (authenticate) { final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); @@ -396,7 +397,7 @@ public void testSnapshotDeleteRestore() { logger.error("--> creating user"); securityClient().preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role", "snapshot_user").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_MAIN_ALIAS); logger.info("--> creating repository"); assertAcked(client().admin().cluster() .preparePutRepository("test-repo") @@ -410,10 +411,10 @@ public void testSnapshotDeleteRestore() { .prepareCreateSnapshot("test-repo", "test-snap-1") .setWaitForCompletion(true) .setIncludeGlobalState(false) - .setIndices(SECURITY_INDEX_NAME) + .setIndices(SECURITY_MAIN_ALIAS) .get().getSnapshotInfo(); assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); - assertThat(snapshotInfo.indices(), contains(SecurityIndexManager.INTERNAL_SECURITY_INDEX)); + assertThat(snapshotInfo.indices(), contains(INTERNAL_SECURITY_MAIN_INDEX_7)); deleteSecurityIndex(); // the realm cache should clear itself but we don't wish to race it securityClient().prepareClearRealmCache().get(); @@ -430,7 +431,7 @@ public void testSnapshotDeleteRestore() { RestoreSnapshotResponse response = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-1") .setWaitForCompletion(true).setIncludeAliases(true).get(); assertThat(response.status(), equalTo(RestStatus.OK)); - assertThat(response.getRestoreInfo().indices(), contains(SecurityIndexManager.INTERNAL_SECURITY_INDEX)); + assertThat(response.getRestoreInfo().indices(), contains(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); // the realm cache should clear itself but we don't wish to race it securityClient().prepareClearRealmCache().get(); // users and roles are retrievable @@ -460,7 +461,7 @@ public void testAuthenticateWithDeletedRole() { .get(); c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get(); logger.error("--> waiting for .security index"); - ensureGreen(SECURITY_INDEX_NAME); + ensureGreen(SECURITY_MAIN_ALIAS); final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster() @@ -591,12 +592,11 @@ public void testUsersAndRolesDoNotInterfereWithIndicesStats() throws Exception { .get(); } - IndicesStatsResponse response = client().admin().indices().prepareStats("foo", SECURITY_INDEX_NAME).get(); + IndicesStatsResponse response = client().admin().indices().prepareStats("foo", SECURITY_MAIN_ALIAS).get(); assertThat(response.getFailedShards(), is(0)); assertThat(response.getIndices().size(), is(2)); - assertThat(response.getIndices().get(INTERNAL_SECURITY_INDEX), notNullValue()); - assertThat(response.getIndices().get(INTERNAL_SECURITY_INDEX).getIndex(), - is(INTERNAL_SECURITY_INDEX)); + assertThat(response.getIndices().get(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7), notNullValue()); + assertThat(response.getIndices().get(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7).getIndex(), is(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); } public void testOperationsOnReservedUsers() throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index fb0d55c75cd32..c9313d20594ae 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -23,7 +23,7 @@ public class NativeRealmTests extends ESTestCase { private final String concreteSecurityIndexName = randomFrom( - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, indexStatus); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index ab82d18f2e0c7..4cbf5307d3ed6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.support.Hasher; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.security.user.APMSystemUser; import org.elasticsearch.xpack.core.security.user.BeatsSystemUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; @@ -111,7 +112,7 @@ public void testBlankPasswordInIndexImpliesDefaultPassword() throws Exception { values.put(PASSWORD_FIELD, BLANK_PASSWORD); final GetResult result = new GetResult( - SecurityIndexManager.SECURITY_INDEX_NAME, + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, MapperService.SINGLE_MAPPING_NAME, NativeUsersStore.getIdForUser(NativeUsersStore.RESERVED_USER_TYPE, randomAlphaOfLength(12)), 0, 1, 1L, @@ -180,7 +181,7 @@ public void testVerifyNonExistentUser() throws Exception { nativeUsersStore.verifyPassword(username, password, future); final GetResult getResult = new GetResult( - SecurityIndexManager.SECURITY_INDEX_NAME, + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, MapperService.SINGLE_MAPPING_NAME, NativeUsersStore.getIdForUser(NativeUsersStore.USER_DOC_TYPE, username), UNASSIGNED_SEQ_NO, 0, 1L, @@ -222,7 +223,7 @@ private void respondToGetUserRequest(String username, SecureString password, Str values.put(User.Fields.TYPE.getPreferredName(), NativeUsersStore.USER_DOC_TYPE); final BytesReference source = BytesReference.bytes(jsonBuilder().map(values)); final GetResult getResult = new GetResult( - SecurityIndexManager.SECURITY_INDEX_NAME, + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, MapperService.SINGLE_MAPPING_NAME, NativeUsersStore.getIdForUser(NativeUsersStore.USER_DOC_TYPE, username), 0, 1, 1L, @@ -230,7 +231,6 @@ private void respondToGetUserRequest(String username, SecureString password, Str source, Collections.emptyMap()); - actionRespond(GetRequest.class, new GetResponse(getResult)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 29407a8672982..4734d9248d820 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -48,7 +48,7 @@ public class NativeRoleMappingStoreTests extends ESTestCase { private final String concreteSecurityIndexName = randomFrom( - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); public void testResolveRoles() throws Exception { // Does match DN diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index 29d02326cd214..c79a7d079567a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -156,8 +156,8 @@ import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionRunAs; import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_SECURITY_INDEX; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; @@ -864,8 +864,8 @@ public void testGrantAllRestrictedUserCannotExecuteOperationAgainstSecurityIndic ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) - .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_MAIN_INDEX_7) + .putAlias(new AliasMetaData.Builder(SECURITY_MAIN_ALIAS).build()) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1) .numberOfReplicas(0) @@ -874,31 +874,33 @@ public void testGrantAllRestrictedUserCannotExecuteOperationAgainstSecurityIndic final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); List> requests = new ArrayList<>(); + requests.add(new Tuple<>(BulkAction.NAME + "[s]", + new DeleteRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id"))); + requests.add( + new Tuple<>(UpdateAction.NAME, new UpdateRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id"))); requests.add( - new Tuple<>(BulkAction.NAME + "[s]", new DeleteRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); - requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); - requests.add(new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); + new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "type", "id"))); + requests.add(new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id"))); requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() - .addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_INDEX)))); + .addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(UpdateSettingsAction.NAME, - new UpdateSettingsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + new UpdateSettingsRequest().indices(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); // cannot execute monitor operations requests.add(new Tuple<>(IndicesStatsAction.NAME, - new IndicesStatsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); - requests.add( - new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + new IndicesStatsRequest().indices(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); + requests.add(new Tuple<>(RecoveryAction.NAME, + new RecoveryRequest().indices(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(IndicesSegmentsAction.NAME, - new IndicesSegmentsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + new IndicesSegmentsRequest().indices(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(GetSettingsAction.NAME, - new GetSettingsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + new GetSettingsRequest().indices(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(IndicesShardStoresAction.NAME, - new IndicesShardStoresRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + new IndicesShardStoresRequest().indices(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(UpgradeStatusAction.NAME, - new UpgradeStatusRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + new UpgradeStatusRequest().indices(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); for (Tuple requestTuple : requests) { String action = requestTuple.v1(); @@ -912,13 +914,13 @@ public void testGrantAllRestrictedUserCannotExecuteOperationAgainstSecurityIndic } // we should allow waiting for the health of the index or any index if the user has this permission - ClusterHealthRequest request = new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)); + ClusterHealthRequest request = new ClusterHealthRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)); authorize(authentication, ClusterHealthAction.NAME, request); verify(auditTrail).accessGranted(eq(requestId), eq(authentication), eq(ClusterHealthAction.NAME), eq(request), authzInfoRoles(new String[]{role.getName()})); // multiple indices - request = new ClusterHealthRequest(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX, "foo", "bar"); + request = new ClusterHealthRequest(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7, "foo", "bar"); authorize(authentication, ClusterHealthAction.NAME, request); verify(auditTrail).accessGranted(eq(requestId), eq(authentication), eq(ClusterHealthAction.NAME), eq(request), authzInfoRoles(new String[]{role.getName()})); @@ -940,8 +942,8 @@ public void testMonitoringOperationsAgainstSecurityIndexRequireAllowRestricted() ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) - .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_MAIN_INDEX_7) + .putAlias(new AliasMetaData.Builder(SECURITY_MAIN_ALIAS).build()) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1) .numberOfReplicas(0) @@ -949,12 +951,12 @@ public void testMonitoringOperationsAgainstSecurityIndexRequireAllowRestricted() .build()); List> requests = new ArrayList<>(); - requests.add(new Tuple<>(IndicesStatsAction.NAME, new IndicesStatsRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(IndicesSegmentsAction.NAME, new IndicesSegmentsRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(IndicesShardStoresAction.NAME, new IndicesShardStoresRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(UpgradeStatusAction.NAME, new UpgradeStatusRequest().indices(SECURITY_INDEX_NAME))); + requests.add(new Tuple<>(IndicesStatsAction.NAME, new IndicesStatsRequest().indices(SECURITY_MAIN_ALIAS))); + requests.add(new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(SECURITY_MAIN_ALIAS))); + requests.add(new Tuple<>(IndicesSegmentsAction.NAME, new IndicesSegmentsRequest().indices(SECURITY_MAIN_ALIAS))); + requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SECURITY_MAIN_ALIAS))); + requests.add(new Tuple<>(IndicesShardStoresAction.NAME, new IndicesShardStoresRequest().indices(SECURITY_MAIN_ALIAS))); + requests.add(new Tuple<>(UpgradeStatusAction.NAME, new UpgradeStatusRequest().indices(SECURITY_MAIN_ALIAS))); for (final Tuple requestTuple : requests) { final String action = requestTuple.v1(); @@ -984,8 +986,8 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndex() throws IOExc ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) - .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_MAIN_INDEX_7) + .putAlias(new AliasMetaData.Builder(SECURITY_MAIN_ALIAS).build()) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1) .numberOfReplicas(0) @@ -994,25 +996,28 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndex() throws IOExc final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); List> requests = new ArrayList<>(); - requests.add(new Tuple<>(DeleteAction.NAME, new DeleteRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); + requests.add( + new Tuple<>(DeleteAction.NAME, new DeleteRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id"))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), DeleteRequest::new))); - requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); - requests.add(new Tuple<>(IndexAction.NAME, new IndexRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + createBulkShardRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), DeleteRequest::new))); + requests.add( + new Tuple<>(UpdateAction.NAME, new UpdateRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id"))); + requests.add(new Tuple<>(IndexAction.NAME, new IndexRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), IndexRequest::new))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + createBulkShardRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), IndexRequest::new))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); - requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); - requests.add(new Tuple<>(IndicesAliasesAction.NAME, - new IndicesAliasesRequest().addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_INDEX)))); + new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "type", "id"))); requests.add( - new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "type", "id"))); + requests.add(new Tuple<>(TermVectorsAction.NAME, + new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "type", "id"))); + requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() + .addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_MAIN_INDEX_7)))); + requests.add(new Tuple<>(ClusterHealthAction.NAME, + new ClusterHealthRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7)))); requests.add(new Tuple<>(ClusterHealthAction.NAME, - new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "foo", "bar"))); + new ClusterHealthRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "foo", "bar"))); for (final Tuple requestTuple : requests) { final String action = requestTuple.v1(); @@ -1033,8 +1038,8 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) - .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_MAIN_INDEX_7) + .putAlias(new AliasMetaData.Builder(INTERNAL_SECURITY_MAIN_INDEX_7).build()) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1) .numberOfReplicas(0) @@ -1046,7 +1051,7 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() SearchRequest request = new SearchRequest("_all"); authorize(authentication, action, request); verify(auditTrail).accessGranted(eq(requestId), eq(authentication), eq(action), eq(request), authzInfoRoles(superuser.roles())); - assertThat(request.indices(), arrayContainingInAnyOrder(INTERNAL_SECURITY_INDEX, SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContainingInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_SECURITY_MAIN_INDEX_7)); } public void testCompositeActionsAreImmediatelyRejected() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java index 202c9cb715f58..51dba4e4c2334 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.List; import java.util.Set; @@ -47,8 +46,8 @@ public void testAuthorizedIndicesUserWithSomeRoles() { RoleDescriptor bRole = new RoleDescriptor("b", null, new IndicesPrivileges[] { IndicesPrivileges.builder().indices("b").privileges("READ").build() }, null); Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); - final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); MetaData metaData = MetaData.builder() .put(new IndexMetaData.Builder("a1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) .put(new IndexMetaData.Builder("a2").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) @@ -65,7 +64,7 @@ public void testAuthorizedIndicesUserWithSomeRoles() { .settings(indexSettings) .numberOfShards(1) .numberOfReplicas(0) - .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build()) + .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build()) .build(), true) .build(); final PlainActionFuture future = new PlainActionFuture<>(); @@ -78,7 +77,7 @@ public void testAuthorizedIndicesUserWithSomeRoles() { assertFalse(list.contains("bbbbb")); assertFalse(list.contains("ba")); assertThat(list, not(contains(internalSecurityIndex))); - assertThat(list, not(contains(RestrictedIndicesNames.SECURITY_INDEX_NAME))); + assertThat(list, not(contains(RestrictedIndicesNames.SECURITY_MAIN_ALIAS))); } public void testAuthorizedIndicesUserWithSomeRolesEmptyMetaData() { @@ -101,8 +100,8 @@ public void testSecurityIndicesAreRestrictedForDefaultRole() { .cluster(ClusterPrivilege.ALL) .build(); Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); - final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); MetaData metaData = MetaData.builder() .put(new IndexMetaData.Builder("an-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) .put(new IndexMetaData.Builder("another-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) @@ -111,7 +110,7 @@ public void testSecurityIndicesAreRestrictedForDefaultRole() { .settings(indexSettings) .numberOfShards(1) .numberOfReplicas(0) - .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build()) + .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build()) .build(), true) .build(); @@ -119,7 +118,7 @@ public void testSecurityIndicesAreRestrictedForDefaultRole() { RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metaData.getAliasAndIndexLookup()); assertThat(authorizedIndices, containsInAnyOrder("an-index", "another-index")); assertThat(authorizedIndices, not(contains(internalSecurityIndex))); - assertThat(authorizedIndices, not(contains(RestrictedIndicesNames.SECURITY_INDEX_NAME))); + assertThat(authorizedIndices, not(contains(RestrictedIndicesNames.SECURITY_MAIN_ALIAS))); } public void testSecurityIndicesAreNotRemovedFromUnrestrictedRole() { @@ -128,8 +127,8 @@ public void testSecurityIndicesAreNotRemovedFromUnrestrictedRole() { .cluster(ClusterPrivilege.ALL) .build(); Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); - final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); MetaData metaData = MetaData.builder() .put(new IndexMetaData.Builder("an-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) .put(new IndexMetaData.Builder("another-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) @@ -137,18 +136,18 @@ public void testSecurityIndicesAreNotRemovedFromUnrestrictedRole() { .settings(indexSettings) .numberOfShards(1) .numberOfReplicas(0) - .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build()) + .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build()) .build(), true) .build(); List authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metaData.getAliasAndIndexLookup()); assertThat(authorizedIndices, containsInAnyOrder( - "an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME, internalSecurityIndex)); + "an-index", "another-index", RestrictedIndicesNames.SECURITY_MAIN_ALIAS, internalSecurityIndex)); List authorizedIndicesSuperUser = RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metaData.getAliasAndIndexLookup()); assertThat(authorizedIndicesSuperUser, containsInAnyOrder( - "an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME, internalSecurityIndex)); + "an-index", "another-index", RestrictedIndicesNames.SECURITY_MAIN_ALIAS, internalSecurityIndex)); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 2f09b74ac3d53..4421c5dc15e5e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -64,7 +64,6 @@ import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -78,7 +77,7 @@ import java.util.Map; import java.util.Set; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.contains; @@ -104,7 +103,6 @@ public class IndicesAndAliasesResolverTests extends ESTestCase { private IndicesAndAliasesResolver defaultIndicesResolver; private IndexNameExpressionResolver indexNameExpressionResolver; private Map roleMap; - private FieldPermissionsCache fieldPermissionsCache; @Before public void setup() { @@ -119,7 +117,7 @@ public void setup() { indexNameExpressionResolver = new IndexNameExpressionResolver(); final boolean withAlias = randomBoolean(); - final String securityIndexName = SECURITY_INDEX_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); + final String securityIndexName = SECURITY_MAIN_ALIAS + (withAlias ? "-" + randomAlphaOfLength(5) : ""); MetaData metaData = MetaData.builder() .put(indexBuilder("foo").putAlias(AliasMetaData.builder("foofoobar")) .putAlias(AliasMetaData.builder("foounauthorized")).settings(settings)) @@ -144,7 +142,6 @@ public void setup() { metaData = SecurityTestUtils.addAliasToMetaData(metaData, securityIndexName); } this.metaData = metaData; - this.fieldPermissionsCache = new FieldPermissionsCache(settings); user = new User("user", "role"); userDashIndices = new User("dash", "dash"); @@ -1220,14 +1217,14 @@ public void testXPackSecurityUserHasAccessToSecurityIndex() { { final List authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(indices, hasItem(SECURITY_MAIN_ALIAS)); } { IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest(); - aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)); + aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_MAIN_ALIAS)); final List authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME); List indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal(); - assertThat(indices, hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(indices, hasItem(SECURITY_MAIN_ALIAS)); } } @@ -1235,7 +1232,7 @@ public void testXPackUserDoesNotHaveAccessToSecurityIndex() { SearchRequest request = new SearchRequest(); final List authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SECURITY_MAIN_ALIAS))); } public void testNonXPackUserAccessingSecurityIndex() { @@ -1247,7 +1244,7 @@ public void testNonXPackUserAccessingSecurityIndex() { SearchRequest request = new SearchRequest(); final List authorizedIndices = buildAuthorizedIndices(allAccessUser, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SECURITY_MAIN_ALIAS))); } { @@ -1255,7 +1252,7 @@ public void testNonXPackUserAccessingSecurityIndex() { aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias1").index("*")); final List authorizedIndices = buildAuthorizedIndices(allAccessUser, IndicesAliasesAction.NAME); List indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SECURITY_MAIN_ALIAS))); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java index 8a79bc86f6702..66ea23b518c29 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java @@ -22,8 +22,8 @@ import java.util.Collections; import java.util.Locale; -import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7; -import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -56,7 +56,7 @@ public void setupClusterBeforeSnapshot() { final String snapshotUserToken = basicAuthHeaderValue(user, new SecureString(password)); client = client().filterWithHeader(Collections.singletonMap("Authorization", snapshotUserToken)); securityClient().preparePutUser(user, password, Hasher.BCRYPT, "snapshot_user").get(); - ensureGreen(INTERNAL_SECURITY_INDEX_7); + ensureGreen(INTERNAL_SECURITY_MAIN_INDEX_7); } public void testSnapshotUserRoleCanSnapshotAndSeeAllIndices() { @@ -67,17 +67,17 @@ public void testSnapshotUserRoleCanSnapshotAndSeeAllIndices() { assertThat(getRepositoriesResponse.repositories().get(0).name(), is("repo")); // view all indices, including restricted ones final GetIndexResponse getIndexResponse = client.admin().indices().prepareGetIndex().setIndices(randomFrom("_all", "*")).get(); - assertThat(Arrays.asList(getIndexResponse.indices()), containsInAnyOrder(INTERNAL_SECURITY_INDEX_7, ordinaryIndex)); + assertThat(Arrays.asList(getIndexResponse.indices()), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, ordinaryIndex)); // create snapshot that includes restricted indices final CreateSnapshotResponse snapshotResponse = client.admin().cluster().prepareCreateSnapshot("repo", "snap") .setIndices(randomFrom("_all", "*")).setWaitForCompletion(true).get(); assertThat(snapshotResponse.getSnapshotInfo().state(), is(SnapshotState.SUCCESS)); - assertThat(snapshotResponse.getSnapshotInfo().indices(), containsInAnyOrder(INTERNAL_SECURITY_INDEX_7, ordinaryIndex)); + assertThat(snapshotResponse.getSnapshotInfo().indices(), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, ordinaryIndex)); // view snapshots for repo final GetSnapshotsResponse getSnapshotResponse = client.admin().cluster().prepareGetSnapshots("repo").get(); assertThat(getSnapshotResponse.getSnapshots().size(), is(1)); assertThat(getSnapshotResponse.getSnapshots().get(0).snapshotId().getName(), is("snap")); - assertThat(getSnapshotResponse.getSnapshots().get(0).indices(), containsInAnyOrder(INTERNAL_SECURITY_INDEX_7, ordinaryIndex)); + assertThat(getSnapshotResponse.getSnapshots().get(0).indices(), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, ordinaryIndex)); } public void testSnapshotUserRoleIsReserved() { @@ -112,7 +112,7 @@ public void testSnapshotUserRoleUnathorizedForDestructiveActions() { () -> client.admin().cluster().prepareDeleteSnapshot("repo", randomAlphaOfLength(4).toLowerCase(Locale.ROOT)).get(), "cluster:admin/snapshot/delete", "snapshot_user"); // try destructive/revealing actions on all indices - for (final String indexToTest : Arrays.asList(INTERNAL_SECURITY_INDEX_7, SECURITY_INDEX_NAME, ordinaryIndex)) { + for (final String indexToTest : Arrays.asList(INTERNAL_SECURITY_MAIN_INDEX_7, SECURITY_MAIN_ALIAS, ordinaryIndex)) { assertThrowsAuthorizationException(() -> client.prepareSearch(indexToTest).get(), "indices:data/read/search", "snapshot_user"); assertThrowsAuthorizationException(() -> client.prepareGet(indexToTest, "doc", "1").get(), "indices:data/read/get", "snapshot_user"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java index 8f3e69815750d..9bd69d3eb1a77 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java @@ -293,14 +293,14 @@ public void testErrorMessageIfIndexPatternIsTooComplex() { public void testSecurityIndicesPermissions() { final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); - final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); final MetaData metaData = new MetaData.Builder() .put(new IndexMetaData.Builder(internalSecurityIndex) .settings(indexSettings) .numberOfShards(1) .numberOfReplicas(0) - .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build()) + .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build()) .build(), true) .build(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); @@ -309,18 +309,18 @@ public void testSecurityIndicesPermissions() { // allow_restricted_indices: false IndicesPermission.Group group = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, false, "*"); Map authzMap = new IndicesPermission(group).authorize(SearchAction.NAME, - Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_INDEX_NAME), lookup, + Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_MAIN_ALIAS), lookup, fieldPermissionsCache); assertThat(authzMap.get(internalSecurityIndex).isGranted(), is(false)); - assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(false)); + assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).isGranted(), is(false)); // allow_restricted_indices: true group = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, true, "*"); authzMap = new IndicesPermission(group).authorize(SearchAction.NAME, - Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_INDEX_NAME), lookup, + Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_MAIN_ALIAS), lookup, fieldPermissionsCache); assertThat(authzMap.get(internalSecurityIndex).isGranted(), is(true)); - assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true)); + assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).isGranted(), is(true)); } private static FieldPermissionsDefinition fieldPermissionDef(String[] granted, String[] denied) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index f17442ca8464e..90d9806655c35 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -96,7 +96,7 @@ public class CompositeRolesStoreTests extends ESTestCase { private final FieldPermissionsCache cache = new FieldPermissionsCache(Settings.EMPTY); private final String concreteSecurityIndexName = randomFrom( - RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7); + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); public void testRolesWhenDlsFlsUnlicensed() throws IOException { XPackLicenseState licenseState = mock(XPackLicenseState.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index 8f60b1d30523f..5dd24856dccd7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequest; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.hamcrest.Matchers; import org.junit.After; @@ -125,7 +126,7 @@ public void testGetSinglePrivilegeByName() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(GetRequest.class)); GetRequest request = (GetRequest) requests.get(0); - assertThat(request.index(), equalTo(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.index(), equalTo(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)); assertThat(request.type(), equalTo(MapperService.SINGLE_MAPPING_NAME)); assertThat(request.id(), equalTo("application-privilege_myapp:admin")); @@ -143,7 +144,7 @@ public void testGetMissingPrivilege() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(GetRequest.class)); GetRequest request = (GetRequest) requests.get(0); - assertThat(request.index(), equalTo(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.index(), equalTo(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)); assertThat(request.type(), equalTo(MapperService.SINGLE_MAPPING_NAME)); assertThat(request.id(), equalTo("application-privilege_myapp:admin")); @@ -166,7 +167,7 @@ public void testGetPrivilegesByApplicationName() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(SearchRequest.class)); SearchRequest request = (SearchRequest) requests.get(0); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)); final String query = Strings.toString(request.source().query()); assertThat(query, containsString("{\"terms\":{\"application\":[\"myapp\",\"yourapp\"]")); @@ -193,7 +194,7 @@ public void testGetAllPrivileges() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(SearchRequest.class)); SearchRequest request = (SearchRequest) requests.get(0); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)); final String query = Strings.toString(request.source().query()); assertThat(query, containsString("{\"term\":{\"type\":{\"value\":\"application-privilege\"")); @@ -229,7 +230,7 @@ public void testPutPrivileges() throws Exception { for (int i = 0; i < putPrivileges.size(); i++) { ApplicationPrivilegeDescriptor privilege = putPrivileges.get(i); IndexRequest request = indexRequests.get(i); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)); assertThat(request.type(), equalTo(MapperService.SINGLE_MAPPING_NAME)); assertThat(request.id(), equalTo( "application-privilege_" + privilege.getApplication() + ":" + privilege.getName() @@ -238,7 +239,7 @@ public void testPutPrivileges() throws Exception { assertThat(request.source(), equalTo(BytesReference.bytes(builder))); final boolean created = privilege.getName().equals("user") == false; indexListener.onResponse(new IndexResponse( - new ShardId(SecurityIndexManager.SECURITY_INDEX_NAME, uuid, i), + new ShardId(RestrictedIndicesNames.SECURITY_MAIN_ALIAS, uuid, i), request.type(), request.id(), 1, 1, 1, created )); } @@ -274,12 +275,12 @@ public void testDeletePrivileges() throws Exception { for (int i = 0; i < privilegeNames.size(); i++) { String name = privilegeNames.get(i); DeleteRequest request = deletes.get(i); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)); assertThat(request.type(), equalTo(MapperService.SINGLE_MAPPING_NAME)); assertThat(request.id(), equalTo("application-privilege_app1:" + name)); final boolean found = name.equals("p2") == false; deleteListener.onResponse(new DeleteResponse( - new ShardId(SecurityIndexManager.SECURITY_INDEX_NAME, uuid, i), + new ShardId(RestrictedIndicesNames.SECURITY_MAIN_ALIAS, uuid, i), request.type(), request.id(), 1, 1, 1, found )); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index d204085a41f79..5f3db4d940412 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -56,7 +56,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -187,7 +187,7 @@ public void testPutOfRoleWithFlsDlsUnlicensed() throws IOException { final ClusterService clusterService = mock(ClusterService.class); final XPackLicenseState licenseState = mock(XPackLicenseState.class); final AtomicBoolean methodCalled = new AtomicBoolean(false); - final SecurityIndexManager securityIndex = SecurityIndexManager.buildSecurityIndexManager(client, clusterService); + final SecurityIndexManager securityIndex = SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService); final NativeRolesStore rolesStore = new NativeRolesStore(Settings.EMPTY, client, licenseState, securityIndex) { @Override void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener listener) { @@ -247,7 +247,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final private ClusterState getClusterStateWithSecurityIndex() { final boolean withAlias = randomBoolean(); - final String securityIndexName = SECURITY_INDEX_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); + final String securityIndexName = SECURITY_MAIN_ALIAS + (withAlias ? "-" + randomAlphaOfLength(5) : ""); Settings settings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) @@ -256,7 +256,7 @@ private ClusterState getClusterStateWithSecurityIndex() { .build(); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder(securityIndexName).settings(settings)) - .put(new IndexTemplateMetaData(SecurityIndexManager.SECURITY_TEMPLATE_NAME, 0, 0, + .put(new IndexTemplateMetaData(SecurityIndexManager.SECURITY_MAIN_TEMPLATE, 0, 0, Collections.singletonList(securityIndexName), Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of())) .build(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 95c5dc96a5059..39ed7da772d49 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -52,12 +52,12 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.security.test.SecurityTestUtils; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.template.TemplateUtils; import org.hamcrest.Matchers; import org.junit.Before; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -71,7 +71,6 @@ public class SecurityIndexManagerTests extends ESTestCase { private static final ClusterName CLUSTER_NAME = new ClusterName("security-index-manager-tests"); private static final ClusterState EMPTY_CLUSTER_STATE = new ClusterState.Builder(CLUSTER_NAME).build(); - private static final String INDEX_NAME = ".security"; private static final String TEMPLATE_NAME = "SecurityIndexManagerTests-template"; private SecurityIndexManager manager; private Map, Map>> actions; @@ -96,13 +95,13 @@ void doExecute(Action action, Request request, ActionListener { @@ -255,8 +255,8 @@ public void testListeneredNotCalledBeforeStateNotRecovered() throws Exception { assertThat(manager.isStateRecovered(), is(false)); assertThat(listenerCalled.get(), is(false)); // state recovered with index - ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME, - SecurityIndexManager.INTERNAL_INDEX_FORMAT); + ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, TEMPLATE_NAME, + SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertThat(manager.isStateRecovered(), is(true)); @@ -278,8 +278,8 @@ public void testIndexOutOfDateListeners() throws Exception { assertTrue(manager.isIndexUpToDate()); // index doesn't exist and now exists with wrong format - ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME, - SecurityIndexManager.INTERNAL_INDEX_FORMAT - 1); + ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, TEMPLATE_NAME, + SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT - 1); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); @@ -295,7 +295,8 @@ public void testIndexOutOfDateListeners() throws Exception { listenerCalled.set(false); // index doesn't exist and now exists with correct format - clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_INDEX_FORMAT); + clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, TEMPLATE_NAME, + SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); @@ -318,7 +319,7 @@ private void assertIndexUpToDateButNotAvailable() { } public static ClusterState.Builder createClusterState(String indexName, String templateName) throws IOException { - return createClusterState(indexName, templateName, templateName, SecurityIndexManager.INTERNAL_INDEX_FORMAT); + return createClusterState(indexName, templateName, templateName, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); } public static ClusterState.Builder createClusterState(String indexName, String templateName, int format) throws IOException { @@ -338,7 +339,7 @@ private static ClusterState.Builder createClusterState(String indexName, String } private void markShardsAvailable(ClusterState.Builder clusterStateBuilder) { - clusterStateBuilder.routingTable(SecurityTestUtils.buildIndexRoutingTable(INDEX_NAME)); + clusterStateBuilder.routingTable(SecurityTestUtils.buildIndexRoutingTable(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); } private static ClusterState state() { @@ -389,7 +390,7 @@ private static String loadTemplate(String templateName) { } public void testMappingVersionMatching() throws IOException { - String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + String templateString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); assertTrue(manager.checkMappingVersion(Version.CURRENT.minimumIndexCompatibilityVersion()::before)); @@ -397,17 +398,19 @@ public void testMappingVersionMatching() throws IOException { } public void testMissingVersionMappingThrowsError() throws IOException { - String templateString = "/missing-version-" + SECURITY_TEMPLATE_NAME + ".json"; + String templateString = "/missing-version-" + SECURITY_MAIN_TEMPLATE + ".json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); final ClusterState clusterState = clusterStateBuilder.build(); IllegalStateException exception = expectThrows(IllegalStateException.class, - () -> SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, logger, Version.CURRENT::equals)); - assertEquals("Cannot read security-version string in index " + SECURITY_INDEX_NAME, exception.getMessage()); + () -> SecurityIndexManager.checkIndexMappingVersionMatches(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + clusterState, logger, Version.CURRENT::equals)); + assertEquals("Cannot read security-version string in index " + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + exception.getMessage()); } public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate( - "/" + SECURITY_TEMPLATE_NAME + ".json" + "/" + SECURITY_MAIN_TEMPLATE + ".json" ); manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); // No upgrade actions run @@ -415,20 +418,20 @@ public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { } public void testIndexTemplateVersionMatching() throws Exception { - String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + String templateString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); final ClusterState clusterState = clusterStateBuilder.build(); assertTrue(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_TEMPLATE_NAME, clusterState, logger, + SecurityIndexManager.SECURITY_MAIN_TEMPLATE, clusterState, logger, Version.V_6_0_0::before)); assertFalse(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_TEMPLATE_NAME, clusterState, logger, + SecurityIndexManager.SECURITY_MAIN_TEMPLATE, clusterState, logger, Version.V_6_0_0::after)); } public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { - String securityTemplateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + String securityTemplateString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(securityTemplateString); manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); @@ -438,8 +441,8 @@ public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { final ClusterName clusterName = new ClusterName("test-cluster"); final ClusterState.Builder clusterStateBuilder = ClusterState.builder(clusterName); - String mappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; - IndexTemplateMetaData.Builder templateMeta = getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, mappingString); + String mappingString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; + IndexTemplateMetaData.Builder templateMeta = getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE, mappingString); MetaData.Builder builder = new MetaData.Builder(clusterStateBuilder.build().getMetaData()); builder.put(templateMeta); clusterStateBuilder.metaData(builder); @@ -450,24 +453,24 @@ public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { private ClusterState.Builder createClusterStateWithTemplate(String securityTemplateString) throws IOException { // add the correct mapping no matter what the template - ClusterState clusterState = createClusterStateWithIndex("/" + SECURITY_TEMPLATE_NAME + ".json").build(); + ClusterState clusterState = createClusterStateWithIndex("/" + SECURITY_MAIN_TEMPLATE + ".json").build(); final MetaData.Builder metaDataBuilder = new MetaData.Builder(clusterState.metaData()); - metaDataBuilder.put(getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, securityTemplateString)); + metaDataBuilder.put(getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE, securityTemplateString)); return ClusterState.builder(clusterState).metaData(metaDataBuilder); } private ClusterState.Builder createClusterStateWithMapping(String securityTemplateString) throws IOException { final ClusterState clusterState = createClusterStateWithIndex(securityTemplateString).build(); final String indexName = clusterState.metaData().getAliasAndIndexLookup() - .get(SECURITY_INDEX_NAME).getIndices().get(0).getIndex().getName(); + .get(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7).getIndices().get(0).getIndex().getName(); return ClusterState.builder(clusterState).routingTable(SecurityTestUtils.buildIndexRoutingTable(indexName)); } private ClusterState.Builder createClusterStateWithMappingAndTemplate(String securityTemplateString) throws IOException { ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(securityTemplateString); MetaData.Builder metaDataBuilder = new MetaData.Builder(clusterStateBuilder.build().metaData()); - String securityMappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; - IndexTemplateMetaData.Builder securityTemplateMeta = getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, securityMappingString); + String securityMappingString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; + IndexTemplateMetaData.Builder securityTemplateMeta = getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE, securityMappingString); metaDataBuilder.put(securityTemplateMeta); return clusterStateBuilder.metaData(metaDataBuilder); } @@ -493,7 +496,8 @@ private static IndexMetaData.Builder createIndexMetadata(String indexName, Strin private ClusterState.Builder createClusterStateWithIndex(String securityTemplate) throws IOException { final MetaData.Builder metaDataBuilder = new MetaData.Builder(); final boolean withAlias = randomBoolean(); - final String securityIndexName = SECURITY_INDEX_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); + final String securityIndexName = RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7 + + (withAlias ? "-" + randomAlphaOfLength(5) : ""); metaDataBuilder.put(createIndexMetadata(securityIndexName, securityTemplate)); ClusterState.Builder clusterStateBuilder = ClusterState.builder(state()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java index 12474b7a04d59..0ee5ff21ee117 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java @@ -35,7 +35,7 @@ import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static java.nio.file.StandardOpenOption.WRITE; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; public class SecurityTestUtils { @@ -83,7 +83,7 @@ public static RoutingTable buildIndexRoutingTable(String indexName) { * Adds the index alias {@code .security} to the underlying concrete index. */ public static MetaData addAliasToMetaData(MetaData metaData, String indexName) { - AliasMetaData aliasMetaData = AliasMetaData.newAliasMetaDataBuilder(SECURITY_INDEX_NAME).build(); + AliasMetaData aliasMetaData = AliasMetaData.newAliasMetaDataBuilder(SECURITY_MAIN_ALIAS).build(); MetaData.Builder metaDataBuilder = new MetaData.Builder(metaData); IndexMetaData indexMetaData = metaData.index(indexName); metaDataBuilder.put(IndexMetaData.builder(indexMetaData).putAlias(aliasMetaData)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java index a295e47b6d704..c8b20e1b4604c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.audit.index.IndexNameResolver; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.hamcrest.Matchers; import org.joda.time.DateTime; @@ -32,8 +31,6 @@ public void testXPackUserCanAccessNonSecurityIndices() { public void testXPackUserCannotAccessRestrictedIndices() { final String action = randomFrom(GetAction.NAME, SearchAction.NAME, IndexAction.NAME); final Predicate predicate = XPackUser.ROLE.indices().allowedIndicesMatcher(action); - assertThat(predicate.test(SecurityIndexManager.SECURITY_INDEX_NAME), Matchers.is(false)); - assertThat(predicate.test(SecurityIndexManager.INTERNAL_SECURITY_INDEX), Matchers.is(false)); for (String index : RestrictedIndicesNames.RESTRICTED_NAMES) { assertThat(predicate.test(index), Matchers.is(false)); } diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 29f3237dcedb5..a670a77281cb3 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.test.StreamsUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.upgrade.UpgradeField; import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder; import org.elasticsearch.xpack.security.support.SecurityIndexManager; @@ -106,7 +107,7 @@ public void testSecurityNativeRealm() throws Exception { if (settingsMap.containsKey("index")) { @SuppressWarnings("unchecked") int format = Integer.parseInt(String.valueOf(((Map)settingsMap.get("index")).get("format"))); - assertEquals("The security index needs to be upgraded", SecurityIndexManager.INTERNAL_INDEX_FORMAT, format); + assertEquals("The security index needs to be upgraded", RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, format); } } diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java index bfec6d100a984..38ce42c6ee9b3 100644 --- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java +++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java @@ -199,7 +199,7 @@ public void cleanupSecurityIndex() throws Exception { @Override public Set excludeTemplates() { Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityIndexManager.SECURITY_TEMPLATE_NAME); // don't remove the security index template + templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE); // don't remove the security index template return templates; } From bf65bd4d360239f20137e5321d6c9fe3f3a313e3 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 28 Mar 2019 15:48:28 +0200 Subject: [PATCH 02/50] WIP TokenService, templates done --- ...te.json => security-index-template-7.json} | 2 +- .../security-tokens-index-template-7.json | 96 +++++++ .../xpack/security/Security.java | 7 +- .../xpack/security/authc/TokenService.java | 271 ++++++++++++------ .../support/SecurityIndexManager.java | 8 +- .../test/NativeRealmIntegTestCase.java | 2 +- .../authz/store/NativeRolesStoreTests.java | 2 +- .../support/SecurityIndexManagerTests.java | 28 +- .../ldap/AbstractAdLdapRealmTestCase.java | 2 +- 9 files changed, 306 insertions(+), 112 deletions(-) rename x-pack/plugin/core/src/main/resources/{security-index-template.json => security-index-template-7.json} (99%) create mode 100644 x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json diff --git a/x-pack/plugin/core/src/main/resources/security-index-template.json b/x-pack/plugin/core/src/main/resources/security-index-template-7.json similarity index 99% rename from x-pack/plugin/core/src/main/resources/security-index-template.json rename to x-pack/plugin/core/src/main/resources/security-index-template-7.json index 94bb2b03ee049..db2e2dcc54552 100644 --- a/x-pack/plugin/core/src/main/resources/security-index-template.json +++ b/x-pack/plugin/core/src/main/resources/security-index-template-7.json @@ -1,5 +1,5 @@ { - "index_patterns" : [ ".security-*" ], + "index_patterns" : [ ".security-7" ], "order" : 1000, "settings" : { "number_of_shards" : 1, diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json new file mode 100644 index 0000000000000..28edfa847f75d --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json @@ -0,0 +1,96 @@ +{ + "index_patterns" : [ ".security-tokens-7" ], + "order" : 1000, + "settings" : { + "number_of_shards" : 1, + "number_of_replicas" : 0, + "auto_expand_replicas" : "0-1", + "index.priority": 1000, + "index.format": 7, + }, + "mappings" : { + "_doc" : { + "_meta": { + "security-version": "${security.template.version}" + }, + "dynamic" : "strict", + "properties" : { + "doc_type" : { + "type" : "keyword" + }, + "creation_time" : { + "type" : "date", + "format" : "epoch_millis" + }, + "refresh_token" : { + "type" : "object", + "properties" : { + "token" : { + "type" : "keyword" + }, + "refreshed" : { + "type" : "boolean" + }, + "refresh_time": { + "type": "date", + "format": "epoch_millis" + }, + "superseded_by": { + "type": "keyword" + }, + "invalidated" : { + "type" : "boolean" + }, + "client" : { + "type" : "object", + "properties" : { + "type" : { + "type" : "keyword" + }, + "user" : { + "type" : "keyword" + }, + "realm" : { + "type" : "keyword" + } + } + } + } + }, + "access_token" : { + "type" : "object", + "properties" : { + "user_token" : { + "type" : "object", + "properties" : { + "id" : { + "type" : "keyword" + }, + "expiration_time" : { + "type" : "date", + "format" : "epoch_millis" + }, + "version" : { + "type" : "integer" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + }, + "authentication" : { + "type" : "binary" + } + } + }, + "invalidated" : { + "type" : "boolean" + }, + "realm" : { + "type" : "keyword" + } + } + } + } + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index dbb0053637857..d909f247c2f9f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -250,7 +250,7 @@ import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7; public class Security extends Plugin implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin, MapperPlugin, ExtensiblePlugin { @@ -398,7 +398,8 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste securityIndex.set(SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService)); - final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService); + final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), + SecurityIndexManager.buildSecurityTokensIndexManager(client, clusterService), clusterService); this.tokenService.set(tokenService); components.add(tokenService); @@ -922,7 +923,7 @@ public List> getExecutorBuilders(final Settings settings) { public UnaryOperator> getIndexTemplateMetaDataUpgrader() { return templates -> { // .security index is not managed by using templates anymore - templates.remove(SECURITY_MAIN_TEMPLATE); + templates.remove(SECURITY_MAIN_TEMPLATE_7); templates.remove("security_audit_log"); return templates; }; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index a9df4c5530e9b..5b7b26d690ff7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.support.master.AcknowledgedRequest; @@ -69,6 +70,7 @@ import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -120,10 +122,12 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -186,7 +190,8 @@ public final class TokenService { private final TimeValue expirationDelay; private final TimeValue deleteInterval; private final Client client; - private final SecurityIndexManager securityIndex; + private final SecurityIndexManager securityMainIndex; + private final SecurityIndexManager securityTokensIndex; private final ExpiredTokenRemover expiredTokenRemover; private final boolean enabled; private volatile TokenKeys keyCache; @@ -200,8 +205,8 @@ public final class TokenService { * @param clock the clock that will be used for comparing timestamps * @param client the client to use when checking for revocations */ - public TokenService(Settings settings, Clock clock, Client client, - SecurityIndexManager securityIndex, ClusterService clusterService) throws GeneralSecurityException { + public TokenService(Settings settings, Clock clock, Client client, SecurityIndexManager securityMainIndex, + SecurityIndexManager securityTokensIndex, ClusterService clusterService) throws GeneralSecurityException { byte[] saltArr = new byte[SALT_BYTES]; secureRandom.nextBytes(saltArr); final SecureString tokenPassphrase = generateTokenKey(); @@ -209,7 +214,8 @@ public TokenService(Settings settings, Clock clock, Client client, this.clock = clock.withZone(ZoneOffset.UTC); this.expirationDelay = TOKEN_EXPIRATION.get(settings); this.client = client; - this.securityIndex = securityIndex; + this.securityMainIndex = securityMainIndex; + this.securityTokensIndex = securityTokensIndex; this.lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); this.deleteInterval = DELETE_INTERVAL.get(settings); this.enabled = isTokenServiceEnabled(settings); @@ -258,46 +264,59 @@ private void createOAuth2Tokens(String userTokenId, Authentication authenticatio authentication.getLookedUpBy(), version, AuthenticationType.TOKEN, authentication.getMetadata()); final UserToken userToken = new UserToken(userTokenId, version, tokenAuth, expiration, metadata); final String documentId = getTokenDocumentId(userToken); - final String refreshToken = includeRefreshToken ? UUIDs.randomBase64UUID() : null; - - final IndexRequest request; - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - builder.field("doc_type", TOKEN_DOC_TYPE); - builder.field("creation_time", created.toEpochMilli()); - if (includeRefreshToken) { - builder.startObject("refresh_token") - .field("token", refreshToken) - .field("invalidated", false) - .field("refreshed", false) - .startObject("client") - .field("type", "unassociated_client") - .field("user", originatingClientAuth.getUser().principal()) - .field("realm", originatingClientAuth.getAuthenticatedBy().getName()) - .endObject() - .endObject(); - } - builder.startObject("access_token") - .field("invalidated", false) - .field("user_token", userToken) - .field("realm", authentication.getAuthenticatedBy().getName()) - .endObject(); - builder.endObject(); - request = client.prepareIndex(securityIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) - .setOpType(OpType.CREATE) - .setSource(builder) - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) - .request(); + final String plainRefreshToken = includeRefreshToken ? UUIDs.randomBase64UUID() : null; + final String versionedRefreshToken = includeRefreshToken ? prependCurrentVersionAndEncode(plainRefreshToken) : null; + + final BytesReference tokenDocument; + try { + tokenDocument = createTokenDocument(created, userToken, plainRefreshToken, authentication, originatingClientAuth); } catch (IOException e) { // unexpected exception listener.onFailure(e); return; } - securityIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", documentId, ex)), - () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, request, ActionListener - .wrap(indexResponse -> { + final IndexRequest indexTokenRequest = client.prepareIndex(securityTokensIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) + .setOpType(OpType.CREATE) + .setSource(tokenDocument, XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) + .request(); + securityTokensIndex.prepareIndexIfNeededThenExecute( + ex -> listener.onFailure(traceLog("prepare security tokens index", documentId, ex)), + () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, indexTokenRequest, + ActionListener.wrap(indexResponse -> { if (indexResponse.getResult() == Result.CREATED) { - listener.onResponse(new Tuple<>(userToken, refreshToken)); + // TODO change upon backport + // create a bwc token document as well + if (clusterService.state().nodes().getMinNodeVersion().before(Version.V_8_0_0)) { + final BytesReference bwcTokenDocument; + try { + bwcTokenDocument = createTokenDocument(created, userToken, versionedRefreshToken, + authentication, originatingClientAuth); + } catch (IOException e) { + // unexpected exception + listener.onFailure(e); + return; + } + final IndexRequest indexBwcTokenRequest = client + .prepareIndex(securityMainIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) + .setOpType(OpType.CREATE) + .setSource(bwcTokenDocument, XContentType.JSON) + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) + .request(); + securityMainIndex.prepareIndexIfNeededThenExecute( + ex -> listener.onFailure(traceLog("prepare security index", documentId, ex)), + () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, + indexBwcTokenRequest, ActionListener.wrap(bwcIndexResponse -> { + if (indexResponse.getResult() == Result.CREATED) { + listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); + } else { + listener.onFailure(traceLog("create token", new ElasticsearchException( + "failed to create token document [{}]", indexResponse))); + } + }, listener::onFailure))); + } else { + listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); + } } else { listener.onFailure(traceLog("create token", new ElasticsearchException("failed to create token document [{}]", indexResponse))); @@ -306,6 +325,35 @@ private void createOAuth2Tokens(String userTokenId, Authentication authenticatio } } + private BytesReference createTokenDocument(Instant created, UserToken userToken, @Nullable String refreshToken, + Authentication authentication, @Nullable Authentication originatingClientAuth) throws IOException { + assert refreshToken == null || originatingClientAuth != null; + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.startObject(); + builder.field("doc_type", TOKEN_DOC_TYPE); + builder.field("creation_time", created.toEpochMilli()); + if (refreshToken != null) { + builder.startObject("refresh_token") + .field("token", refreshToken) + .field("invalidated", false) + .field("refreshed", false) + .startObject("client") + .field("type", "unassociated_client") + .field("user", originatingClientAuth.getUser().principal()) + .field("realm", originatingClientAuth.getAuthenticatedBy().getName()) + .endObject() + .endObject(); + } + builder.startObject("access_token") + .field("invalidated", false) + .field("user_token", userToken) + .field("realm", authentication.getAuthenticatedBy().getName()) + .endObject(); + builder.endObject(); + return BytesReference.bytes(builder); + } + } + /** * Looks in the context to see if the request provided a header with a user token and if so the * token is validated, which might include authenticated decryption and verification that the token @@ -350,15 +398,21 @@ public void getAuthenticationAndMetaData(String token, ActionListener listener) { - if (securityIndex.isAvailable() == false) { - logger.warn("failed to get access token [{}] because index is not available", userTokenId); + void getUserTokenFromId(String userTokenId, Version version, ActionListener listener) { + final SecurityIndexManager tokensIndexManager; + if (version.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport + tokensIndexManager = securityTokensIndex; + } else { + tokensIndexManager = securityMainIndex; + } + if (tokensIndexManager.isAvailable() == false) { + logger.warn("failed to get access token [{}] because index [{}] is not available", userTokenId, tokensIndexManager.aliasName()); listener.onResponse(null); } else { - securityIndex.checkIndexVersionThenExecute( - ex -> listener.onFailure(traceLog("prepare security index", userTokenId, ex)), + tokensIndexManager.checkIndexVersionThenExecute( + ex -> listener.onFailure(traceLog("prepare security tokens index", userTokenId, ex)), () -> { - final GetRequest getRequest = client.prepareGet(securityIndex.aliasName(), SINGLE_MAPPING_NAME, + final GetRequest getRequest = client.prepareGet(tokensIndexManager.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userTokenId)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("decode token", userTokenId, ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, @@ -385,7 +439,8 @@ void getUserTokenFromId(String userTokenId, ActionListener listener) // if the index or the shard is not there / available we assume that // the token is not valid if (isShardNotAvailableException(e)) { - logger.warn("failed to get access token [{}] because index is not available", userTokenId); + logger.warn("failed to get access token [{}] because index [{}] is not available", userTokenId, + tokensIndexManager.aliasName()); listener.onResponse(null); } else { logger.error(new ParameterizedMessage("failed to get access token [{}]", userTokenId), e); @@ -411,7 +466,7 @@ void decodeToken(String token, ActionListener listener) { if (version.onOrAfter(Version.V_7_1_0)) { // The token was created in a > 7.1.0 cluster so it contains the tokenId as a String String usedTokenId = in.readString(); - getUserTokenFromId(usedTokenId, listener); + getUserTokenFromId(usedTokenId, version, listener); } else { // The token was created in a < 7.1.0 cluster so we need to decrypt it to get the tokenId if (in.available() < MINIMUM_BASE64_BYTES) { @@ -432,7 +487,7 @@ void decodeToken(String token, ActionListener listener) { try { final Cipher cipher = getDecryptionCipher(iv, decodeKey, version, decodedSalt); final String tokenId = decryptTokenId(encryptedTokenId, cipher, version); - getUserTokenFromId(tokenId, listener); + getUserTokenFromId(tokenId, version, listener); } catch (IOException | GeneralSecurityException e) { // could happen with a token that is not ours logger.warn("invalid token", e); @@ -476,7 +531,7 @@ public void invalidateAccessToken(String accessToken, ActionListener listener) { ensureEnabled(); @@ -495,12 +548,12 @@ public void invalidateAccessToken(UserToken userToken, ActionListener backoff = DEFAULT_BACKOFF.iterator(); - indexInvalidation(Collections.singleton(userToken.getId()), backoff, "access_token", null, listener); + indexInvalidation(Collections.singleton(userToken), backoff, "access_token", null, listener); } } /** - * This method onvalidates a refresh token so that it may no longer be used. Iinvalidation involves performing an update to the token + * This method invalidates a refresh token so that it may no longer be used. Invalidation involves performing an update to the token * document and setting the refresh_token.invalidated field to true * * @param refreshToken The string representation of the refresh token @@ -543,7 +596,7 @@ public void invalidateActiveTokensForRealmAndUser(@Nullable String realmName, @N logger.warn("No tokens to invalidate for realm [{}] and username [{}]", realmName, username); listener.onResponse(TokensInvalidationResult.emptyResult()); } else { - invalidateAllTokens(tokenTuples.stream().map(t -> t.v1().getId()).collect(Collectors.toList()), listener); + invalidateAllTokens(tokenTuples.stream().map(t -> t.v1()).collect(Collectors.toList()), listener); } }, listener::onFailure)); } else { @@ -556,7 +609,7 @@ public void invalidateActiveTokensForRealmAndUser(@Nullable String realmName, @N logger.warn("No tokens to invalidate for realm [{}] and username [{}]", realmName, username); listener.onResponse(TokensInvalidationResult.emptyResult()); } else { - invalidateAllTokens(tokenTuples.stream().map(t -> t.v1().getId()).collect(Collectors.toList()), listener); + invalidateAllTokens(tokenTuples.stream().map(t -> t.v1()).collect(Collectors.toList()), listener); } }, listener::onFailure)); } @@ -567,34 +620,64 @@ public void invalidateActiveTokensForRealmAndUser(@Nullable String realmName, @N * Invalidates a collection of access_token and refresh_token that were retrieved by * {@link TokenService#invalidateActiveTokensForRealmAndUser} * - * @param accessTokenIds The ids of the access tokens which should be invalidated (along with the respective refresh_token) + * @param userTokens The the user tokens for which access and refresh tokens should be invalidated * @param listener the listener to notify upon completion */ - private void invalidateAllTokens(Collection accessTokenIds, ActionListener listener) { + private void invalidateAllTokens(Collection userTokens, ActionListener listener) { maybeStartTokenRemover(); // Invalidate the refresh tokens first so that they cannot be used to get new // access tokens while we invalidate the access tokens we currently know about final Iterator backoff = DEFAULT_BACKOFF.iterator(); - indexInvalidation(accessTokenIds, backoff, "refresh_token", null, ActionListener.wrap(result -> - indexInvalidation(accessTokenIds, backoff, "access_token", result, listener), + indexInvalidation(userTokens, backoff, "refresh_token", null, ActionListener.wrap(result -> + indexInvalidation(userTokens, backoff, "access_token", result, listener), listener::onFailure)); } + /** + * Invalidates access and/or refresh tokens associated to a user token (coexisting in the same token document) + */ + private void indexInvalidation(Collection userTokens, Iterator backoff, String srcPrefix, + @Nullable TokensInvalidationResult previousResult, ActionListener listener) { + final Set idsOfRecentTokens = new HashSet<>(); + final Set idsOfOlderTokens = new HashSet<>(); + for (UserToken userToken : userTokens) { + if (userToken.getVersion().onOrAfter(Version.V_8_0_0)) { // change upon backport + idsOfRecentTokens.add(userToken.getId()); + } else { + idsOfOlderTokens.add(userToken.getId()); + } + } + if (false == idsOfOlderTokens.isEmpty()) { + indexInvalidation(idsOfOlderTokens, securityMainIndex, backoff, srcPrefix, null, ActionListener.wrap(result -> { + if (false == idsOfRecentTokens.isEmpty()) { + // carry-over result of the invalidation for the main security index + indexInvalidation(idsOfRecentTokens, securityTokensIndex, backoff, srcPrefix, result, listener); + } else { + listener.onResponse(result); + } + }, listener::onFailure)); + } else { + indexInvalidation(idsOfRecentTokens, securityTokensIndex, backoff, srcPrefix, null, listener); + } + } + /** * Performs the actual invalidation of a collection of tokens. In case of recoverable errors ( see * {@link TransportActions#isShardNotAvailableException} ) the UpdateRequests to mark the tokens as invalidated are retried using * an exponential backoff policy. * - * @param tokenIds the tokens to invalidate - * @param backoff the amount of time to delay between attempts - * @param srcPrefix the prefix to use when constructing the doc to update, either refresh_token or access_token depending on - * what type of tokens should be invalidated - * @param previousResult if this not the initial attempt for invalidation, it contains the result of invalidating - * tokens up to the point of the retry. This result is added to the result of the current attempt - * @param listener the listener to notify upon completion + * @param tokenIds the tokens to invalidate + * @param tokensIndexManager the manager for the index where the tokens are stored + * @param backoff the amount of time to delay between attempts + * @param srcPrefix the prefix to use when constructing the doc to update, either refresh_token or access_token depending on + * what type of tokens should be invalidated + * @param previousResult if this not the initial attempt for invalidation, it contains the result of invalidating + * tokens up to the point of the retry. This result is added to the result of the current attempt + * @param listener the listener to notify upon completion */ - private void indexInvalidation(Collection tokenIds, Iterator backoff, String srcPrefix, - @Nullable TokensInvalidationResult previousResult, ActionListener listener) { + private void indexInvalidation(Collection tokenIds, SecurityIndexManager tokensIndexManager, Iterator backoff, + String srcPrefix, @Nullable TokensInvalidationResult previousResult, + ActionListener listener) { if (tokenIds.isEmpty()) { logger.warn("No [{}] tokens provided for invalidation", srcPrefix); listener.onFailure(invalidGrantException("No tokens provided for invalidation")); @@ -602,14 +685,14 @@ private void indexInvalidation(Collection tokenIds, Iterator BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); for (String tokenId : tokenIds) { UpdateRequest request = client - .prepareUpdate(securityIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(tokenId)) + .prepareUpdate(tokensIndexManager.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(tokenId)) .setDoc(srcPrefix, Collections.singletonMap("invalidated", true)) .setFetchSource(srcPrefix, null) .request(); bulkRequestBuilder.add(request); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - securityIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", ex)), + tokensIndexManager.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", ex)), () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequestBuilder.request(), ActionListener.wrap(bulkResponse -> { ArrayList retryTokenDocIds = new ArrayList<>(); @@ -647,8 +730,9 @@ private void indexInvalidation(Collection tokenIds, Iterator retryTokenDocIds.size(), tokenIds.size()); final TokensInvalidationResult incompleteResult = new TokensInvalidationResult(invalidated, previouslyInvalidated, failedRequestResponses); - final Runnable retryWithContextRunnable = client.threadPool().getThreadContext().preserveContext( - () -> indexInvalidation(retryTokenDocIds, backoff, srcPrefix, incompleteResult, listener)); + final Runnable retryWithContextRunnable = client.threadPool().getThreadContext() + .preserveContext(() -> indexInvalidation(retryTokenDocIds, tokensIndexManager, backoff, + srcPrefix, incompleteResult, listener)); client.threadPool().schedule(retryWithContextRunnable, backoff.next(), GENERIC); } else { if (retryTokenDocIds.isEmpty() == false) { @@ -670,7 +754,8 @@ private void indexInvalidation(Collection tokenIds, Iterator if (isShardNotAvailableException(cause) && backoff.hasNext()) { logger.debug("failed to invalidate tokens, retrying "); final Runnable retryWithContextRunnable = client.threadPool().getThreadContext() - .preserveContext(() -> indexInvalidation(tokenIds, backoff, srcPrefix, previousResult, listener)); + .preserveContext(() -> indexInvalidation(tokenIds, tokensIndexManager, backoff, srcPrefix, + previousResult, listener)); client.threadPool().schedule(retryWithContextRunnable, backoff.next(), GENERIC); } else { listener.onFailure(e); @@ -713,7 +798,7 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator onFailure.accept(ex); } }; - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityMainIndex.freeze(); if (frozenSecurityIndex.indexExists() == false) { logger.warn("security index does not exist therefore refresh token cannot be validated"); listener.onFailure(invalidGrantException("could not refresh the requested token")); @@ -721,13 +806,14 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator logger.debug("security index is not available to find token from refresh token, retrying"); maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); } else { - final SearchRequest request = client.prepareSearch(securityIndex.aliasName()) + final SearchRequest request = client.prepareSearch(securityMainIndex.aliasName()) .setQuery(QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) .filter(QueryBuilders.termQuery("refresh_token.token", refreshToken))) .seqNoAndPrimaryTerm(true) + .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN) .request(); - securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> + securityMainIndex.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, ActionListener.wrap(searchResponse -> { if (searchResponse.isTimedOut()) { @@ -838,7 +924,7 @@ public void onFailure(Exception e) { updateMap.put("superseded_by", getTokenDocumentId(newUserTokenId)); assert seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; assert primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM : "expected an assigned primary term"; - final UpdateRequestBuilder updateRequest = client.prepareUpdate(securityIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) + final UpdateRequestBuilder updateRequest = client.prepareUpdate(securityMainIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) .setDoc("refresh_token", updateMap) .setFetchSource(true) .setRefreshPolicy(RefreshPolicy.IMMEDIATE) @@ -918,7 +1004,7 @@ public void onFailure(Exception e) { } private void getTokenDocAsync(String tokenDocId, ActionListener listener) { - final GetRequest getRequest = client.prepareGet(securityIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId).request(); + final GetRequest getRequest = client.prepareGet(securityMainIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, listener, client::get); } @@ -1036,7 +1122,7 @@ private static Optional checkMultipleRefreshes(I public void findActiveTokensForRealm(String realmName, @Nullable Predicate> filter, ActionListener>> listener) { ensureEnabled(); - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityMainIndex.freeze(); if (Strings.isNullOrEmpty(realmName)) { listener.onFailure(new IllegalArgumentException("Realm name is required")); } else if (frozenSecurityIndex.indexExists() == false) { @@ -1059,14 +1145,14 @@ public void findActiveTokensForRealm(String realmName, @Nullable Predicate ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, filter))); } } @@ -1080,7 +1166,7 @@ public void findActiveTokensForRealm(String realmName, @Nullable Predicate>> listener) { ensureEnabled(); - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityMainIndex.freeze(); if (Strings.isNullOrEmpty(username)) { listener.onFailure(new IllegalArgumentException("username is required")); } else if (frozenSecurityIndex.indexExists() == false) { @@ -1102,14 +1188,14 @@ public void findActiveTokensForUser(String username, ActionListener ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, isOfUser(username)))); } @@ -1187,14 +1273,14 @@ private void checkIfTokenIsValid(UserToken userToken, ActionListener Instant currentTime = clock.instant(); if (currentTime.isAfter(userToken.getExpirationTime())) { listener.onFailure(traceLog("validate token", userToken.getId(), expiredTokenException())); - } else if (securityIndex.indexExists() == false) { + } else if (securityMainIndex.indexExists() == false) { // index doesn't exist so the token is considered invalid as we cannot verify its validity logger.warn("failed to validate access token because the security index doesn't exist"); listener.onResponse(null); } else { - securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { + securityMainIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { final GetRequest getRequest = client - .prepareGet(securityIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userToken)).request(); + .prepareGet(securityMainIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userToken)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("check token state", userToken.getId(), ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { @@ -1240,7 +1326,7 @@ private Instant getExpirationTime(Instant now) { } private void maybeStartTokenRemover() { - if (securityIndex.isAvailable()) { + if (securityMainIndex.isAvailable()) { if (client.threadPool().relativeTimeInMillis() - lastExpirationRunMs > deleteInterval.getMillis()) { expiredTokenRemover.submit(client.threadPool()); lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); @@ -1300,6 +1386,17 @@ public String getAccessTokenAsString(UserToken userToken) throws IOException, Ge } } + private static String prependCurrentVersionAndEncode(String payload) { + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + OutputStream base64 = Base64.getEncoder().wrap(os); + StreamOutput out = new OutputStreamStreamOutput(base64)) { + out.setVersion(Version.CURRENT); + Version.writeVersion(Version.CURRENT, out); + out.writeString(payload); + return new String(os.toByteArray(), StandardCharsets.UTF_8); + } + } + private void ensureEncryptionCiphersSupported() throws NoSuchPaddingException, NoSuchAlgorithmException { Cipher.getInstance(ENCRYPTION_CIPHER); SecretKeyFactory.getInstance(KDF_ALGORITHM); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 468a9a535c5f9..d9ae21e31614b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -79,8 +79,8 @@ public class SecurityIndexManager implements ClusterStateListener { public static final int INTERNAL_MAIN_INDEX_FORMAT = 6; public static final int INTERNAL_TOKENS_INDEX_FORMAT = 7; - public static final String SECURITY_MAIN_TEMPLATE = "security-index-template"; - public static final String SECURITY_TOKENS_TEMPLATE = "security-tokens-index-template"; + public static final String SECURITY_MAIN_TEMPLATE_7 = "security-index-template-7"; + public static final String SECURITY_TOKENS_TEMPLATE_7 = "security-tokens-index-template-7"; public static final String SECURITY_VERSION_STRING = "security-version"; public static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); @@ -99,13 +99,13 @@ public class SecurityIndexManager implements ClusterStateListener { public static SecurityIndexManager buildSecurityMainIndexManager(Client client, ClusterService clusterService) { return new SecurityIndexManager(client, clusterService, RestrictedIndicesNames.SECURITY_MAIN_ALIAS, RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_MAIN_INDEX_FORMAT, - () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_MAIN_TEMPLATE)); + () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_MAIN_TEMPLATE_7)); } public static SecurityIndexManager buildSecurityTokensIndexManager(Client client, ClusterService clusterService) { return new SecurityIndexManager(client, clusterService, RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, INTERNAL_TOKENS_INDEX_FORMAT, - () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_TOKENS_TEMPLATE)); + () -> SecurityIndexManager.readTemplateAsBytes(SECURITY_TOKENS_TEMPLATE_7)); } private SecurityIndexManager(Client client, ClusterService clusterService, String aliasName, String internalIndexName, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index e08ea8b6022f2..671a94452fa0a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -59,7 +59,7 @@ protected boolean addMockHttpTransport() { @Override public Set excludeTemplates() { Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE); // don't remove the security index template + templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7); // don't remove the security index template return templates; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index 5f3db4d940412..39c65fb87822e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -256,7 +256,7 @@ private ClusterState getClusterStateWithSecurityIndex() { .build(); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder(securityIndexName).settings(settings)) - .put(new IndexTemplateMetaData(SecurityIndexManager.SECURITY_MAIN_TEMPLATE, 0, 0, + .put(new IndexTemplateMetaData(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, 0, 0, Collections.singletonList(securityIndexName), Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of())) .build(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 39ed7da772d49..4b6581d41a9eb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -57,7 +57,7 @@ import org.hamcrest.Matchers; import org.junit.Before; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -390,7 +390,7 @@ private static String loadTemplate(String templateName) { } public void testMappingVersionMatching() throws IOException { - String templateString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; + String templateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); assertTrue(manager.checkMappingVersion(Version.CURRENT.minimumIndexCompatibilityVersion()::before)); @@ -398,7 +398,7 @@ public void testMappingVersionMatching() throws IOException { } public void testMissingVersionMappingThrowsError() throws IOException { - String templateString = "/missing-version-" + SECURITY_MAIN_TEMPLATE + ".json"; + String templateString = "/missing-version-" + SECURITY_MAIN_TEMPLATE_7 + ".json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); final ClusterState clusterState = clusterStateBuilder.build(); IllegalStateException exception = expectThrows(IllegalStateException.class, @@ -410,7 +410,7 @@ public void testMissingVersionMappingThrowsError() throws IOException { public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate( - "/" + SECURITY_MAIN_TEMPLATE + ".json" + "/" + SECURITY_MAIN_TEMPLATE_7 + ".json" ); manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); // No upgrade actions run @@ -418,20 +418,20 @@ public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { } public void testIndexTemplateVersionMatching() throws Exception { - String templateString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; + String templateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); final ClusterState clusterState = clusterStateBuilder.build(); assertTrue(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_MAIN_TEMPLATE, clusterState, logger, + SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, clusterState, logger, Version.V_6_0_0::before)); assertFalse(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityIndexManager.SECURITY_MAIN_TEMPLATE, clusterState, logger, + SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7, clusterState, logger, Version.V_6_0_0::after)); } public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { - String securityTemplateString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; + String securityTemplateString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(securityTemplateString); manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); @@ -441,8 +441,8 @@ public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { final ClusterName clusterName = new ClusterName("test-cluster"); final ClusterState.Builder clusterStateBuilder = ClusterState.builder(clusterName); - String mappingString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; - IndexTemplateMetaData.Builder templateMeta = getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE, mappingString); + String mappingString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; + IndexTemplateMetaData.Builder templateMeta = getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE_7, mappingString); MetaData.Builder builder = new MetaData.Builder(clusterStateBuilder.build().getMetaData()); builder.put(templateMeta); clusterStateBuilder.metaData(builder); @@ -453,9 +453,9 @@ public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { private ClusterState.Builder createClusterStateWithTemplate(String securityTemplateString) throws IOException { // add the correct mapping no matter what the template - ClusterState clusterState = createClusterStateWithIndex("/" + SECURITY_MAIN_TEMPLATE + ".json").build(); + ClusterState clusterState = createClusterStateWithIndex("/" + SECURITY_MAIN_TEMPLATE_7 + ".json").build(); final MetaData.Builder metaDataBuilder = new MetaData.Builder(clusterState.metaData()); - metaDataBuilder.put(getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE, securityTemplateString)); + metaDataBuilder.put(getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE_7, securityTemplateString)); return ClusterState.builder(clusterState).metaData(metaDataBuilder); } @@ -469,8 +469,8 @@ private ClusterState.Builder createClusterStateWithMapping(String securityTempla private ClusterState.Builder createClusterStateWithMappingAndTemplate(String securityTemplateString) throws IOException { ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(securityTemplateString); MetaData.Builder metaDataBuilder = new MetaData.Builder(clusterStateBuilder.build().metaData()); - String securityMappingString = "/" + SECURITY_MAIN_TEMPLATE + ".json"; - IndexTemplateMetaData.Builder securityTemplateMeta = getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE, securityMappingString); + String securityMappingString = "/" + SECURITY_MAIN_TEMPLATE_7 + ".json"; + IndexTemplateMetaData.Builder securityTemplateMeta = getIndexTemplateMetaData(SECURITY_MAIN_TEMPLATE_7, securityMappingString); metaDataBuilder.put(securityTemplateMeta); return clusterStateBuilder.metaData(metaDataBuilder); } diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java index 38ce42c6ee9b3..86d411053afe9 100644 --- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java +++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java @@ -199,7 +199,7 @@ public void cleanupSecurityIndex() throws Exception { @Override public Set excludeTemplates() { Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE); // don't remove the security index template + templates.add(SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7); // don't remove the security index template return templates; } From b2dd6b80b7ac0a82eb0e1cb5a1b3fe562f356c1b Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 28 Mar 2019 22:16:34 +0200 Subject: [PATCH 03/50] WIP getting closer --- .../xpack/security/authc/TokenService.java | 165 +++++++++++------- 1 file changed, 99 insertions(+), 66 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 5b7b26d690ff7..8082f7e6a977e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -30,7 +30,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.support.master.AcknowledgedRequest; @@ -200,10 +199,6 @@ public final class TokenService { /** * Creates a new token service - * - * @param settings the node settings - * @param clock the clock that will be used for comparing timestamps - * @param client the client to use when checking for revocations */ public TokenService(Settings settings, Clock clock, Client client, SecurityIndexManager securityMainIndex, SecurityIndexManager securityTokensIndex, ClusterService clusterService) throws GeneralSecurityException { @@ -259,16 +254,16 @@ private void createOAuth2Tokens(String userTokenId, Authentication authenticatio } else { final Instant created = clock.instant(); final Instant expiration = getExpirationTime(created); - final Version version = clusterService.state().nodes().getMinNodeVersion(); + final Version minNodeVersion = clusterService.state().nodes().getMinNodeVersion(); final Authentication tokenAuth = new Authentication(authentication.getUser(), authentication.getAuthenticatedBy(), - authentication.getLookedUpBy(), version, AuthenticationType.TOKEN, authentication.getMetadata()); - final UserToken userToken = new UserToken(userTokenId, version, tokenAuth, expiration, metadata); + authentication.getLookedUpBy(), minNodeVersion, AuthenticationType.TOKEN, authentication.getMetadata()); + final UserToken userToken = new UserToken(userTokenId, minNodeVersion, tokenAuth, expiration, metadata); final String documentId = getTokenDocumentId(userToken); final String plainRefreshToken = includeRefreshToken ? UUIDs.randomBase64UUID() : null; - final String versionedRefreshToken = includeRefreshToken ? prependCurrentVersionAndEncode(plainRefreshToken) : null; - + final String versionedRefreshToken; final BytesReference tokenDocument; try { + versionedRefreshToken = includeRefreshToken ? prependVersionAndEncode(minNodeVersion, plainRefreshToken) : null; tokenDocument = createTokenDocument(created, userToken, plainRefreshToken, authentication, originatingClientAuth); } catch (IOException e) { // unexpected exception @@ -287,7 +282,7 @@ private void createOAuth2Tokens(String userTokenId, Authentication authenticatio if (indexResponse.getResult() == Result.CREATED) { // TODO change upon backport // create a bwc token document as well - if (clusterService.state().nodes().getMinNodeVersion().before(Version.V_8_0_0)) { + if (minNodeVersion.before(Version.V_8_0_0)) { final BytesReference bwcTokenDocument; try { bwcTokenDocument = createTokenDocument(created, userToken, versionedRefreshToken, @@ -325,35 +320,6 @@ private void createOAuth2Tokens(String userTokenId, Authentication authenticatio } } - private BytesReference createTokenDocument(Instant created, UserToken userToken, @Nullable String refreshToken, - Authentication authentication, @Nullable Authentication originatingClientAuth) throws IOException { - assert refreshToken == null || originatingClientAuth != null; - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - builder.field("doc_type", TOKEN_DOC_TYPE); - builder.field("creation_time", created.toEpochMilli()); - if (refreshToken != null) { - builder.startObject("refresh_token") - .field("token", refreshToken) - .field("invalidated", false) - .field("refreshed", false) - .startObject("client") - .field("type", "unassociated_client") - .field("user", originatingClientAuth.getUser().principal()) - .field("realm", originatingClientAuth.getAuthenticatedBy().getName()) - .endObject() - .endObject(); - } - builder.startObject("access_token") - .field("invalidated", false) - .field("user_token", userToken) - .field("realm", authentication.getAuthenticatedBy().getName()) - .endObject(); - builder.endObject(); - return BytesReference.bytes(builder); - } - } - /** * Looks in the context to see if the request provided a header with a user token and if so the * token is validated, which might include authenticated decryption and verification that the token @@ -399,12 +365,7 @@ public void getAuthenticationAndMetaData(String token, ActionListener listener) { - final SecurityIndexManager tokensIndexManager; - if (version.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport - tokensIndexManager = securityTokensIndex; - } else { - tokensIndexManager = securityMainIndex; - } + final SecurityIndexManager tokensIndexManager = getSecurityIndexManagerForVersion(version); if (tokensIndexManager.isAvailable() == false) { logger.warn("failed to get access token [{}] because index [{}] is not available", userTokenId, tokensIndexManager.aliasName()); listener.onResponse(null); @@ -569,8 +530,8 @@ public void invalidateRefreshToken(String refreshToken, ActionListener backoff = DEFAULT_BACKOFF.iterator(); findTokenFromRefreshToken(refreshToken, backoff, ActionListener.wrap(searchResponse -> { - final String docId = getTokenIdFromDocumentId(searchResponse.getId()); - indexInvalidation(Collections.singletonList(docId), backoff, "refresh_token", null, listener); + final Tuple parsedTokens = parseTokensFromDocument(searchResponse.getSourceAsMap(), null); + indexInvalidation(Collections.singletonList(parsedTokens.v1()), backoff, "refresh_token", null, listener); }, listener::onFailure)); } } @@ -782,38 +743,56 @@ public void refreshToken(String refreshToken, ActionListener backoff, ActionListener listener) { + Tuple versionAndRefreshTokenTuple; + try { + versionAndRefreshTokenTuple = unpackVersionAndPayload(refreshToken); + } catch (IOException e) { + // could be an old format refresh token + logger.debug("refresh token could be one of the old unversioned format"); + versionAndRefreshTokenTuple = new Tuple(Version.V_7_0_0, refreshToken); + } + final SecurityIndexManager tokensIndexManager = getSecurityIndexManagerForVersion(versionAndRefreshTokenTuple.v1()); + findTokenFromRefreshToken(versionAndRefreshTokenTuple.v2(), tokensIndexManager, backoff, listener); + } + + /** + * Performs an asynchronous search request for the token document that contains the {@code refreshToken} and calls the listener with the + * {@link SearchResponse}. In case of recoverable errors the {@code SearchRequest} is retried using an exponential backoff policy. This method + * requires the tokens index where the token document, pointed to by the refresh token, resides. + */ + private void findTokenFromRefreshToken(String refreshToken, SecurityIndexManager tokensIndexManager, Iterator backoff, + ActionListener listener) { final Consumer onFailure = ex -> listener.onFailure(traceLog("find token by refresh token", refreshToken, ex)); final Consumer maybeRetryOnFailure = ex -> { if (backoff.hasNext()) { final TimeValue backofTimeValue = backoff.next(); logger.debug("retrying after [" + backofTimeValue + "] back off"); final Runnable retryWithContextRunnable = client.threadPool().getThreadContext() - .preserveContext(() -> findTokenFromRefreshToken(refreshToken, backoff, listener)); + .preserveContext(() -> findTokenFromRefreshToken(refreshToken, tokensIndexManager, backoff, listener)); client.threadPool().schedule(retryWithContextRunnable, backofTimeValue, GENERIC); } else { logger.warn("failed to find token from refresh token after all retries"); onFailure.accept(ex); } }; - final SecurityIndexManager frozenSecurityIndex = securityMainIndex.freeze(); - if (frozenSecurityIndex.indexExists() == false) { - logger.warn("security index does not exist therefore refresh token cannot be validated"); + final SecurityIndexManager frozenTokensIndex = tokensIndexManager.freeze(); + if (frozenTokensIndex.indexExists() == false) { + logger.warn("index [{}] does not exist therefore refresh token cannot be validated", frozenTokensIndex.aliasName()); listener.onFailure(invalidGrantException("could not refresh the requested token")); - } else if (frozenSecurityIndex.isAvailable() == false) { - logger.debug("security index is not available to find token from refresh token, retrying"); + } else if (frozenTokensIndex.isAvailable() == false) { + logger.debug("index [{}] is not available to find token from refresh token, retrying", frozenTokensIndex.aliasName()); maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); } else { - final SearchRequest request = client.prepareSearch(securityMainIndex.aliasName()) + final SearchRequest request = client.prepareSearch(tokensIndexManager.aliasName()) .setQuery(QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) .filter(QueryBuilders.termQuery("refresh_token.token", refreshToken))) .seqNoAndPrimaryTerm(true) - .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN) .request(); - securityMainIndex.checkIndexVersionThenExecute(listener::onFailure, () -> + tokensIndexManager.checkIndexVersionThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, ActionListener.wrap(searchResponse -> { if (searchResponse.isTimedOut()) { @@ -1201,6 +1180,36 @@ public void findActiveTokensForUser(String username, ActionListener> isOfUser(String username) { return source -> { String auth = (String) source.get("authentication"); @@ -1266,21 +1275,35 @@ private void ensureEnabled() { } } + /** + * In version 7.1 tokens moved into a separate index away from other objects. But they moved "seamlessly" - without manual intervention. + * In this way, new tokens were created in the new index, while existing ones were left in place, to be removed by the + * {@code ExpiredTokenRemover}. But when searching for a token we need to consider both indices. + */ + private SecurityIndexManager getSecurityIndexManagerForVersion(Version version) { + if (version.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport + return securityTokensIndex; + } else { + return securityMainIndex; + } + } + /** * Checks if the access token has been explicitly invalidated */ private void checkIfTokenIsValid(UserToken userToken, ActionListener listener) { - Instant currentTime = clock.instant(); - if (currentTime.isAfter(userToken.getExpirationTime())) { + if (clock.instant().isAfter(userToken.getExpirationTime())) { listener.onFailure(traceLog("validate token", userToken.getId(), expiredTokenException())); - } else if (securityMainIndex.indexExists() == false) { + } + final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(userToken.getVersion()); + if (tokensIndex.indexExists() == false) { // index doesn't exist so the token is considered invalid as we cannot verify its validity logger.warn("failed to validate access token because the security index doesn't exist"); listener.onResponse(null); } else { - securityMainIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { + tokensIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { final GetRequest getRequest = client - .prepareGet(securityMainIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userToken)).request(); + .prepareGet(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userToken)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("check token state", userToken.getId(), ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { @@ -1386,17 +1409,27 @@ public String getAccessTokenAsString(UserToken userToken) throws IOException, Ge } } - private static String prependCurrentVersionAndEncode(String payload) { + private static String prependVersionAndEncode(Version version, String payload) throws IOException { try (ByteArrayOutputStream os = new ByteArrayOutputStream(); OutputStream base64 = Base64.getEncoder().wrap(os); StreamOutput out = new OutputStreamStreamOutput(base64)) { - out.setVersion(Version.CURRENT); - Version.writeVersion(Version.CURRENT, out); + out.setVersion(version); + Version.writeVersion(version, out); out.writeString(payload); return new String(os.toByteArray(), StandardCharsets.UTF_8); } } + private static Tuple unpackVersionAndPayload(String encodedPack) throws IOException { + final byte[] bytes = encodedPack.getBytes(StandardCharsets.UTF_8); + try (StreamInput in = new InputStreamStreamInput(Base64.getDecoder().wrap(new ByteArrayInputStream(bytes)), bytes.length)) { + final Version version = Version.readVersion(in); + in.setVersion(version); + final String payload = in.readString(); + return new Tuple(version, payload); + } + } + private void ensureEncryptionCiphersSupported() throws NoSuchPaddingException, NoSuchAlgorithmException { Cipher.getInstance(ENCRYPTION_CIPHER); SecretKeyFactory.getInstance(KDF_ALGORITHM); From 27d5679ab301f543bd72829e7168cde14d61cb96 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 28 Mar 2019 22:49:26 +0200 Subject: [PATCH 04/50] InnerRefresh is next, then find*, then token remover --- .../xpack/security/authc/TokenService.java | 57 +++++-------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 8082f7e6a977e..bfb1dbe74b38c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -264,54 +264,27 @@ private void createOAuth2Tokens(String userTokenId, Authentication authenticatio final BytesReference tokenDocument; try { versionedRefreshToken = includeRefreshToken ? prependVersionAndEncode(minNodeVersion, plainRefreshToken) : null; - tokenDocument = createTokenDocument(created, userToken, plainRefreshToken, authentication, originatingClientAuth); + if (minNodeVersion.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport + tokenDocument = createTokenDocument(created, userToken, plainRefreshToken, authentication, originatingClientAuth); + } else { + tokenDocument = createTokenDocument(created, userToken, versionedRefreshToken, authentication, originatingClientAuth); + } } catch (IOException e) { // unexpected exception listener.onFailure(e); return; } - final IndexRequest indexTokenRequest = client.prepareIndex(securityTokensIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) + final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(minNodeVersion); + final IndexRequest indexTokenRequest = client.prepareIndex(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) .setOpType(OpType.CREATE) .setSource(tokenDocument, XContentType.JSON) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .request(); - securityTokensIndex.prepareIndexIfNeededThenExecute( - ex -> listener.onFailure(traceLog("prepare security tokens index", documentId, ex)), + tokensIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security tokens index", documentId, ex)), () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, indexTokenRequest, ActionListener.wrap(indexResponse -> { if (indexResponse.getResult() == Result.CREATED) { - // TODO change upon backport - // create a bwc token document as well - if (minNodeVersion.before(Version.V_8_0_0)) { - final BytesReference bwcTokenDocument; - try { - bwcTokenDocument = createTokenDocument(created, userToken, versionedRefreshToken, - authentication, originatingClientAuth); - } catch (IOException e) { - // unexpected exception - listener.onFailure(e); - return; - } - final IndexRequest indexBwcTokenRequest = client - .prepareIndex(securityMainIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) - .setOpType(OpType.CREATE) - .setSource(bwcTokenDocument, XContentType.JSON) - .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) - .request(); - securityMainIndex.prepareIndexIfNeededThenExecute( - ex -> listener.onFailure(traceLog("prepare security index", documentId, ex)), - () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, - indexBwcTokenRequest, ActionListener.wrap(bwcIndexResponse -> { - if (indexResponse.getResult() == Result.CREATED) { - listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); - } else { - listener.onFailure(traceLog("create token", new ElasticsearchException( - "failed to create token document [{}]", indexResponse))); - } - }, listener::onFailure))); - } else { - listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); - } + listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); } else { listener.onFailure(traceLog("create token", new ElasticsearchException("failed to create token document [{}]", indexResponse))); @@ -365,15 +338,15 @@ public void getAuthenticationAndMetaData(String token, ActionListener listener) { - final SecurityIndexManager tokensIndexManager = getSecurityIndexManagerForVersion(version); - if (tokensIndexManager.isAvailable() == false) { - logger.warn("failed to get access token [{}] because index [{}] is not available", userTokenId, tokensIndexManager.aliasName()); + final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(version); + if (tokensIndex.isAvailable() == false) { + logger.warn("failed to get access token [{}] because index [{}] is not available", userTokenId, tokensIndex.aliasName()); listener.onResponse(null); } else { - tokensIndexManager.checkIndexVersionThenExecute( + tokensIndex.checkIndexVersionThenExecute( ex -> listener.onFailure(traceLog("prepare security tokens index", userTokenId, ex)), () -> { - final GetRequest getRequest = client.prepareGet(tokensIndexManager.aliasName(), SINGLE_MAPPING_NAME, + final GetRequest getRequest = client.prepareGet(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userTokenId)).request(); Consumer onFailure = ex -> listener.onFailure(traceLog("decode token", userTokenId, ex)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, @@ -401,7 +374,7 @@ void getUserTokenFromId(String userTokenId, Version version, ActionListener Date: Mon, 1 Apr 2019 14:06:54 +0300 Subject: [PATCH 05/50] TokenService done --- .../security/authc/ExpiredTokenRemover.java | 47 +++- .../xpack/security/authc/TokenService.java | 223 ++++++++++++------ .../support/SecurityIndexManager.java | 39 +-- 3 files changed, 208 insertions(+), 101 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index de099ed02a9a8..69ba7ce49e337 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.DeleteByQueryAction; @@ -22,10 +21,12 @@ import org.elasticsearch.index.reindex.ScrollableHitSource; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool.Names; -import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException; @@ -37,21 +38,28 @@ * The document gets deleted if it was created more than 24 hours which is the maximum * lifetime of a refresh token */ -final class ExpiredTokenRemover extends AbstractRunnable { +final class ExpiredTokenRemover { private static final Logger logger = LogManager.getLogger(ExpiredTokenRemover.class); + public static final long MAXIMUM_TOKEN_LIFETIME_HOURS = 24L; + private final Client client; - private final AtomicBoolean inProgress = new AtomicBoolean(false); + private final SecurityIndexManager securityMainIndex; + private final SecurityIndexManager securityTokensIndex; + private final AtomicBoolean inProgress; private final TimeValue timeout; - ExpiredTokenRemover(Settings settings, Client client) { + ExpiredTokenRemover(Settings settings, Client client, SecurityIndexManager securityMainIndex, + SecurityIndexManager securityTokensIndex) { this.client = client; + this.securityMainIndex = securityMainIndex; + this.securityTokensIndex = securityTokensIndex; + this.inProgress = new AtomicBoolean(false); this.timeout = TokenService.DELETE_TIMEOUT.get(settings); } - @Override - public void doRun() { - DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(RestrictedIndicesNames.SECURITY_MAIN_ALIAS); + public void doRun(String... tokensIndexNames) { + DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(tokensIndexNames); if (timeout != TimeValue.MINUS_ONE) { expiredDbq.setTimeout(timeout); expiredDbq.getSearchRequest().source().timeout(timeout); @@ -59,8 +67,9 @@ public void doRun() { final Instant now = Instant.now(); expiredDbq .setQuery(QueryBuilders.boolQuery() - .filter(QueryBuilders.termsQuery("doc_type", "token")) - .filter(QueryBuilders.rangeQuery("creation_time").lte(now.minus(24L, ChronoUnit.HOURS).toEpochMilli()))); + .filter(QueryBuilders.termsQuery("doc_type", TokenService.TOKEN_DOC_TYPE)) + .filter(QueryBuilders.rangeQuery("creation_time") + .lte(now.minus(MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS).toEpochMilli()))); logger.trace(() -> new ParameterizedMessage("Removing old tokens: [{}]", Strings.toString(expiredDbq))); executeAsyncWithOrigin(client, SECURITY_ORIGIN, DeleteByQueryAction.INSTANCE, expiredDbq, ActionListener.wrap(r -> { @@ -69,10 +78,23 @@ public void doRun() { }, this::onFailure)); } - void submit(ThreadPool threadPool) { + boolean submit(ThreadPool threadPool) { + final List indicesWithTokens = new ArrayList<>(); + if (securityTokensIndex.isAvailable()) { + indicesWithTokens.add(securityTokensIndex.aliasName()); + } + if (securityMainIndex.isAvailable() && (false == securityTokensIndex.indexExists() + || Instant.now().minus(MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS).isBefore(securityTokensIndex.getCreationTime()))) { + indicesWithTokens.add(securityMainIndex.aliasName()); + } + if (indicesWithTokens.isEmpty()) { + return false; + } if (inProgress.compareAndSet(false, true)) { - threadPool.executor(Names.GENERIC).submit(this); + threadPool.executor(Names.GENERIC).submit(() -> doRun(indicesWithTokens.toArray(new String[0]))); + return true; } + return false; } private void debugDbqResponse(BulkByScrollResponse response) { @@ -94,7 +116,6 @@ boolean isExpirationInProgress() { return inProgress.get(); } - @Override public void onFailure(Exception e) { if (isShardNotAvailableException(e)) { logger.debug("failed to delete expired tokens", e); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index bfb1dbe74b38c..831853c5d1b85 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -176,7 +176,7 @@ public final class TokenService { public static final Setting DELETE_TIMEOUT = Setting.timeSetting("xpack.security.authc.token.delete.timeout", TimeValue.MINUS_ONE, Property.NodeScope); - private static final String TOKEN_DOC_TYPE = "token"; + static final String TOKEN_DOC_TYPE = "token"; private static final String TOKEN_DOC_ID_PREFIX = TOKEN_DOC_TYPE + "_"; static final int MINIMUM_BYTES = VERSION_BYTES + SALT_BYTES + IV_BYTES + 1; static final int MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * MINIMUM_BYTES) / 3)).intValue(); @@ -214,7 +214,7 @@ public TokenService(Settings settings, Clock clock, Client client, SecurityIndex this.lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); this.deleteInterval = DELETE_INTERVAL.get(settings); this.enabled = isTokenServiceEnabled(settings); - this.expiredTokenRemover = new ExpiredTokenRemover(settings, client); + this.expiredTokenRemover = new ExpiredTokenRemover(settings, client, securityMainIndex, securityTokensIndex); ensureEncryptionCiphersSupported(); KeyAndCache keyAndCache = new KeyAndCache(new KeyAndTimestamp(tokenPassphrase, createdTimeStamps.incrementAndGet()), new BytesKey(saltArr)); @@ -235,16 +235,20 @@ public static Boolean isTokenServiceEnabled(Settings settings) { public void createOAuth2Tokens(Authentication authentication, Authentication originatingClientAuth, Map metadata, boolean includeRefreshToken, ActionListener> listener) { - createOAuth2Tokens(UUIDs.randomBase64UUID(), authentication, originatingClientAuth, metadata, includeRefreshToken, listener); + // the created token is compatible with the oldest node version in the cluster + final Version minNodeVersion = clusterService.state().nodes().getMinNodeVersion(); + // the id of the created tokens ought be unguessable + final String userTokenId = UUIDs.randomBase64UUID(); + createOAuth2Tokens(userTokenId, minNodeVersion, authentication, originatingClientAuth, metadata, includeRefreshToken, listener); } /** * Create an access token and optionally a refresh token as well, based on the provided authentication and metadata, with the given * token document id. The created tokens are be stored in the security index. */ - private void createOAuth2Tokens(String userTokenId, Authentication authentication, Authentication originatingClientAuth, - Map metadata, boolean includeRefreshToken, - ActionListener> listener) { + private void createOAuth2Tokens(String userTokenId, Version version, Authentication authentication, + Authentication originatingClientAuth, Map metadata, + boolean includeRefreshToken, ActionListener> listener) { ensureEnabled(); if (authentication == null) { listener.onFailure(traceLog("create token", new IllegalArgumentException("authentication must be provided"))); @@ -252,29 +256,19 @@ private void createOAuth2Tokens(String userTokenId, Authentication authenticatio listener.onFailure(traceLog("create token", new IllegalArgumentException("originating client authentication must be provided"))); } else { - final Instant created = clock.instant(); - final Instant expiration = getExpirationTime(created); - final Version minNodeVersion = clusterService.state().nodes().getMinNodeVersion(); final Authentication tokenAuth = new Authentication(authentication.getUser(), authentication.getAuthenticatedBy(), - authentication.getLookedUpBy(), minNodeVersion, AuthenticationType.TOKEN, authentication.getMetadata()); - final UserToken userToken = new UserToken(userTokenId, minNodeVersion, tokenAuth, expiration, metadata); - final String documentId = getTokenDocumentId(userToken); + authentication.getLookedUpBy(), version, AuthenticationType.TOKEN, authentication.getMetadata()); + final UserToken userToken = new UserToken(userTokenId, version, tokenAuth, getExpirationTime(), metadata); final String plainRefreshToken = includeRefreshToken ? UUIDs.randomBase64UUID() : null; - final String versionedRefreshToken; + final String versionedRefreshToken = includeRefreshToken ? prependVersionAndEncode(version, plainRefreshToken) : null; final BytesReference tokenDocument; - try { - versionedRefreshToken = includeRefreshToken ? prependVersionAndEncode(minNodeVersion, plainRefreshToken) : null; - if (minNodeVersion.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport - tokenDocument = createTokenDocument(created, userToken, plainRefreshToken, authentication, originatingClientAuth); - } else { - tokenDocument = createTokenDocument(created, userToken, versionedRefreshToken, authentication, originatingClientAuth); - } - } catch (IOException e) { - // unexpected exception - listener.onFailure(e); - return; + if (version.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport + tokenDocument = createTokenDocument(userToken, plainRefreshToken, originatingClientAuth); + } else { + tokenDocument = createTokenDocument(userToken, versionedRefreshToken, originatingClientAuth); } - final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(minNodeVersion); + final String documentId = getTokenDocumentId(userToken); + final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(version); final IndexRequest indexTokenRequest = client.prepareIndex(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) .setOpType(OpType.CREATE) .setSource(tokenDocument, XContentType.JSON) @@ -819,13 +813,13 @@ private void innerRefresh(String tokenDocId, Map source, long se if (refreshTokenStatus.isRefreshed()) { logger.debug("Token document [{}] was recently refreshed, when a new token document [{}] was generated. Reusing that result.", tokenDocId, refreshTokenStatus.getSupersedingDocId()); - getTokenDocAsync(refreshTokenStatus.getSupersedingDocId(), new ActionListener() { + getSupersedingTokenDocAsync(refreshTokenStatus, new ActionListener() { private final Consumer maybeRetryOnFailure = ex -> { if (backoff.hasNext()) { final TimeValue backofTimeValue = backoff.next(); logger.debug("retrying after [" + backofTimeValue + "] back off"); final Runnable retryWithContextRunnable = client.threadPool().getThreadContext() - .preserveContext(() -> getTokenDocAsync(refreshTokenStatus.getSupersedingDocId(), this)); + .preserveContext(() -> getSupersedingTokenDocAsync(refreshTokenStatus, this)); client.threadPool().schedule(retryWithContextRunnable, backofTimeValue, GENERIC); } else { logger.warn("back off retries exhausted"); @@ -870,13 +864,19 @@ public void onFailure(Exception e) { }); } else { final String newUserTokenId = UUIDs.randomBase64UUID(); + final Version newTokenVersion = clusterService.state().nodes().getMinNodeVersion(); final Map updateMap = new HashMap<>(); updateMap.put("refreshed", true); updateMap.put("refresh_time", clock.instant().toEpochMilli()); - updateMap.put("superseded_by", getTokenDocumentId(newUserTokenId)); + if (newTokenVersion.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport + updateMap.put("superseded_by", prependVersionAndEncode(newTokenVersion, getTokenDocumentId(newUserTokenId))); + } else { + updateMap.put("superseded_by", getTokenDocumentId(newUserTokenId)); + } assert seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; assert primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM : "expected an assigned primary term"; - final UpdateRequestBuilder updateRequest = client.prepareUpdate(securityMainIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) + final SecurityIndexManager refreshedTokenIndex = getSecurityIndexManagerForVersion(refreshTokenStatus.getVersion()); + final UpdateRequestBuilder updateRequest = client.prepareUpdate(refreshedTokenIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) .setDoc("refresh_token", updateMap) .setFetchSource(true) .setRefreshPolicy(RefreshPolicy.IMMEDIATE) @@ -889,7 +889,7 @@ public void onFailure(Exception e) { updateResponse.getGetResult().sourceAsMap())); final Tuple parsedTokens = parseTokensFromDocument(source, null); final UserToken toRefreshUserToken = parsedTokens.v1(); - createOAuth2Tokens(newUserTokenId, toRefreshUserToken.getAuthentication(), clientAuth, + createOAuth2Tokens(newUserTokenId, newTokenVersion, toRefreshUserToken.getAuthentication(), clientAuth, toRefreshUserToken.getMetadata(), true, listener); } else if (backoff.hasNext()) { logger.info("failed to update the original token document [{}], the update result was [{}]. Retrying", @@ -908,7 +908,7 @@ public void onFailure(Exception e) { if (cause instanceof VersionConflictEngineException) { // The document has been updated by another thread, get it again. logger.debug("version conflict while updating document [{}], attempting to get it again", tokenDocId); - getTokenDocAsync(tokenDocId, new ActionListener() { + getTokenDocAsync(tokenDocId, refreshedTokenIndex, new ActionListener() { @Override public void onResponse(GetResponse response) { if (response.isExists()) { @@ -926,7 +926,7 @@ public void onFailure(Exception e) { if (backoff.hasNext()) { logger.info("could not get token document [{}] for refresh, retrying", tokenDocId); final Runnable retryWithContextRunnable = client.threadPool().getThreadContext() - .preserveContext(() -> getTokenDocAsync(tokenDocId, this)); + .preserveContext(() -> getTokenDocAsync(tokenDocId, refreshedTokenIndex, this)); client.threadPool().schedule(retryWithContextRunnable, backoff.next(), GENERIC); } else { logger.warn("could not get token document [{}] for refresh after all retries", tokenDocId); @@ -955,14 +955,27 @@ public void onFailure(Exception e) { } } - private void getTokenDocAsync(String tokenDocId, ActionListener listener) { - final GetRequest getRequest = client.prepareGet(securityMainIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId).request(); + private void getSupersedingTokenDocAsync(RefreshTokenStatus refreshTokenStatus, ActionListener listener) { + Tuple versionAndSupersedingTokenDocId; + try { + versionAndSupersedingTokenDocId = unpackVersionAndPayload(refreshTokenStatus.getSupersedingDocId()); + } catch (IOException e) { + // could be an old format refresh token + logger.debug("superseding token-doc-id in old unversioned format"); + versionAndSupersedingTokenDocId = new Tuple(Version.V_7_0_0, refreshTokenStatus.getSupersedingDocId()); + } + final SecurityIndexManager securityIndexManager = getSecurityIndexManagerForVersion(versionAndSupersedingTokenDocId.v1()); + getTokenDocAsync(versionAndSupersedingTokenDocId.v2(), securityIndexManager, listener); + } + + private void getTokenDocAsync(String tokenDocId, SecurityIndexManager tokensIndex, ActionListener listener) { + final GetRequest getRequest = client.prepareGet(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, listener, client::get); } /** - * A refresh token has a hardcoded maximum lifetime of 24h. This checks if the token document represents a valid token wrt this time - * interval. + * A refresh token has a fixed maximum lifetime of {@code ExpiredTokenRemover#MAXIMUM_TOKEN_LIFETIME_HOURS} hours. This checks if the + * token document represents a valid token wrt this time interval. */ private static Optional checkTokenDocumentExpired(Instant now, Map source) { final Long creationEpochMilli = (Long) source.get("creation_time"); @@ -970,7 +983,7 @@ private static Optional checkTokenDocumentExpire throw new IllegalStateException("token document is missing creation time value"); } else { final Instant creationTime = Instant.ofEpochMilli(creationEpochMilli); - if (now.isAfter(creationTime.plus(24L, ChronoUnit.HOURS))) { + if (now.isAfter(creationTime.plus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS))) { return Optional.of(invalidGrantException("token document has expired")); } else { return Optional.empty(); @@ -987,12 +1000,13 @@ private static Tuple source) throws IllegalStateException, DateTimeException { final RefreshTokenStatus refreshTokenStatus = RefreshTokenStatus.fromSourceMap(getRefreshTokenSourceMap(source)); final UserToken userToken = UserToken.fromSourceMap(getUserTokenSourceMap(source)); + refreshTokenStatus.setVersion(userToken.getVersion()); final ElasticsearchSecurityException validationException = checkTokenDocumentExpired(now, source).orElseGet(() -> { if (refreshTokenStatus.isInvalidated()) { return invalidGrantException("token has been invalidated"); } else { return checkClientCanRefresh(refreshTokenStatus, clientAuth) - .orElse(checkMultipleRefreshes(now, refreshTokenStatus, userToken).orElse(null)); + .orElse(checkMultipleRefreshes(now, refreshTokenStatus).orElse(null)); } }); return new Tuple<>(refreshTokenStatus, Optional.ofNullable(validationException)); @@ -1045,10 +1059,9 @@ private static Map getUserTokenSourceMap(Map sou * @return An {@code Optional} containing the exception in case this refresh token cannot be reused, or an empty Optional if * refreshing is allowed. */ - private static Optional checkMultipleRefreshes(Instant now, RefreshTokenStatus refreshToken, - UserToken userToken) { + private static Optional checkMultipleRefreshes(Instant now, RefreshTokenStatus refreshToken) { if (refreshToken.isRefreshed()) { - if (userToken.getVersion().onOrAfter(Version.V_7_1_0)) { + if (refreshToken.getVersion().onOrAfter(Version.V_7_1_0)) { if (now.isAfter(refreshToken.getRefreshInstant().plus(30L, ChronoUnit.SECONDS))) { return Optional.of(invalidGrantException("token has already been refreshed more than 30 seconds in the past")); } @@ -1074,15 +1087,39 @@ private static Optional checkMultipleRefreshes(I public void findActiveTokensForRealm(String realmName, @Nullable Predicate> filter, ActionListener>> listener) { ensureEnabled(); - final SecurityIndexManager frozenSecurityIndex = securityMainIndex.freeze(); if (Strings.isNullOrEmpty(realmName)) { listener.onFailure(new IllegalArgumentException("Realm name is required")); - } else if (frozenSecurityIndex.indexExists() == false) { + return; + } + final Instant now = clock.instant(); + findActiveTokensForRealm(realmName, securityTokensIndex, now, filter, ActionListener.wrap(tokens -> { + if (false == securityTokensIndex.indexExists() + || Instant.now().minus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS) + .isBefore(securityTokensIndex.getCreationTime())) { + findActiveTokensForRealm(realmName, securityMainIndex, now, filter, ActionListener.wrap(tokensInSecurityMain -> { + if (false == tokensInSecurityMain.isEmpty()) { + final List> allTokens = new ArrayList<>(); + allTokens.addAll(tokens); + allTokens.addAll(tokensInSecurityMain); + listener.onResponse(Collections.unmodifiableList(allTokens)); + } else { + listener.onResponse(tokens); + } + }, listener::onFailure)); + } else { + listener.onResponse(tokens); + } + }, listener::onFailure)); + } + private void findActiveTokensForRealm(String realmName, SecurityIndexManager tokensIndex, Instant now, + @Nullable Predicate> filter, + ActionListener>> listener) { + final SecurityIndexManager frozenTokensIndex = tokensIndex.freeze(); + if (frozenTokensIndex.indexExists() == false) { listener.onResponse(Collections.emptyList()); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenTokensIndex.isAvailable() == false) { + listener.onFailure(frozenTokensIndex.getUnavailableReason()); } else { - final Instant now = clock.instant(); final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) .filter(QueryBuilders.termQuery("access_token.realm", realmName)) @@ -1093,18 +1130,18 @@ public void findActiveTokensForRealm(String realmName, @Nullable Predicate ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, filter))); } } @@ -1118,15 +1155,39 @@ public void findActiveTokensForRealm(String realmName, @Nullable Predicate>> listener) { ensureEnabled(); - final SecurityIndexManager frozenSecurityIndex = securityMainIndex.freeze(); if (Strings.isNullOrEmpty(username)) { listener.onFailure(new IllegalArgumentException("username is required")); - } else if (frozenSecurityIndex.indexExists() == false) { + return; + } + final Instant now = clock.instant(); + findActiveTokensForUser(username, securityTokensIndex, now, ActionListener.wrap(tokens -> { + if (false == securityTokensIndex.indexExists() + || Instant.now().minus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS) + .isBefore(securityTokensIndex.getCreationTime())) { + findActiveTokensForUser(username, securityMainIndex, now, ActionListener.wrap(tokensInSecurityMain -> { + if (false == tokensInSecurityMain.isEmpty()) { + final List> allTokens = new ArrayList<>(); + allTokens.addAll(tokens); + allTokens.addAll(tokensInSecurityMain); + listener.onResponse(Collections.unmodifiableList(allTokens)); + } else { + listener.onResponse(tokens); + } + }, listener::onFailure)); + } else { + listener.onResponse(tokens); + } + }, listener::onFailure)); + } + + private void findActiveTokensForUser(String username, SecurityIndexManager tokensIndex, Instant now, + ActionListener>> listener) { + final SecurityIndexManager frozenTokensIndex = tokensIndex.freeze(); + if (frozenTokensIndex.indexExists() == false) { listener.onResponse(Collections.emptyList()); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenTokensIndex.isAvailable() == false) { + listener.onFailure(frozenTokensIndex.getUnavailableReason()); } else { - final Instant now = clock.instant(); final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) .filter(QueryBuilders.boolQuery() @@ -1136,31 +1197,30 @@ public void findActiveTokensForUser(String username, ActionListener ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, isOfUser(username)))); } } - private static BytesReference createTokenDocument(Instant created, UserToken userToken, @Nullable String refreshToken, - Authentication authentication,@Nullable Authentication originatingClientAuth) - throws IOException { + private BytesReference createTokenDocument(UserToken userToken, @Nullable String refreshToken, + @Nullable Authentication originatingClientAuth) { assert refreshToken == null || originatingClientAuth != null; try (XContentBuilder builder = XContentFactory.jsonBuilder()) { builder.startObject(); builder.field("doc_type", TOKEN_DOC_TYPE); - builder.field("creation_time", created.toEpochMilli()); + builder.field("creation_time", getCreationTime(userToken.getExpirationTime()).toEpochMilli()); if (refreshToken != null) { builder.startObject("refresh_token") .field("token", refreshToken) @@ -1176,10 +1236,12 @@ private static BytesReference createTokenDocument(Instant created, UserToken use builder.startObject("access_token") .field("invalidated", false) .field("user_token", userToken) - .field("realm", authentication.getAuthenticatedBy().getName()) + .field("realm", userToken.getAuthentication().getAuthenticatedBy().getName()) .endObject(); builder.endObject(); return BytesReference.bytes(builder); + } catch (IOException e) { + throw new RuntimeException("Unexpected exception when constructing a JSON document.", e); } } @@ -1249,9 +1311,10 @@ private void ensureEnabled() { } /** - * In version 7.1 tokens moved into a separate index away from other objects. But they moved "seamlessly" - without manual intervention. - * In this way, new tokens were created in the new index, while existing ones were left in place, to be removed by the - * {@code ExpiredTokenRemover}. But when searching for a token we need to consider both indices. + * In version 7.1 tokens moved into a separate index away from the other entities due to their ephemeral nature. But they moved + * "seamlessly" - without manual intervention. In this way, new tokens are created in the new index, while the existing ones were left + * in place - to be accessed from the old index - and due to be removed automatically by the {@code ExpiredTokenRemover} periodic job. + * Therefore, in general, when searching for a token we need to consider both the new and the old indices. */ private SecurityIndexManager getSecurityIndexManagerForVersion(Version version) { if (version.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport @@ -1317,14 +1380,17 @@ public TimeValue getExpirationDelay() { return expirationDelay; } - private Instant getExpirationTime(Instant now) { - return now.plusSeconds(expirationDelay.getSeconds()); + private Instant getExpirationTime() { + return clock.instant().plusSeconds(expirationDelay.getSeconds()); + } + + private Instant getCreationTime(Instant expire) { + return expire.minusSeconds(expirationDelay.getSeconds()); } private void maybeStartTokenRemover() { - if (securityMainIndex.isAvailable()) { - if (client.threadPool().relativeTimeInMillis() - lastExpirationRunMs > deleteInterval.getMillis()) { - expiredTokenRemover.submit(client.threadPool()); + if (client.threadPool().relativeTimeInMillis() - lastExpirationRunMs > deleteInterval.getMillis()) { + if (expiredTokenRemover.submit(client.threadPool())) { lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); } } @@ -1382,7 +1448,7 @@ public String getAccessTokenAsString(UserToken userToken) throws IOException, Ge } } - private static String prependVersionAndEncode(Version version, String payload) throws IOException { + private static String prependVersionAndEncode(Version version, String payload) { try (ByteArrayOutputStream os = new ByteArrayOutputStream(); OutputStream base64 = Base64.getEncoder().wrap(os); StreamOutput out = new OutputStreamStreamOutput(base64)) { @@ -1390,6 +1456,8 @@ private static String prependVersionAndEncode(Version version, String payload) t Version.writeVersion(version, out); out.writeString(payload); return new String(os.toByteArray(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException("Unexpected exception when working with small in memory streams", e); } } @@ -1941,6 +2009,7 @@ private static final class RefreshTokenStatus { private final boolean refreshed; @Nullable private final Instant refreshInstant; @Nullable private final String supersededByDocId; + private Version version; private RefreshTokenStatus(boolean invalidated, String associatedUser, String associatedRealm, boolean refreshed, Instant refreshInstant, String supersededByDocId) { @@ -1976,6 +2045,14 @@ boolean isRefreshed() { return supersededByDocId; } + Version getVersion() { + return version; + } + + void setVersion(Version version) { + this.version = version; + } + static RefreshTokenStatus fromSourceMap(Map refreshTokenSource) { final Boolean invalidated = (Boolean) refreshTokenSource.get("invalidated"); if (invalidated == null) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index d9ae21e31614b..254527e5f71f9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -139,7 +140,11 @@ public String aliasName() { } public boolean indexExists() { - return this.indexState.indexExists; + return this.indexState.indexExists(); + } + + public Instant getCreationTime() { + return this.indexState.creationTime; } /** @@ -168,7 +173,7 @@ public ElasticsearchException getUnavailableReason() { throw new IllegalStateException("caller must make sure to use a frozen state and check indexAvailable"); } - if (localState.indexExists) { + if (localState.indexExists()) { return new UnavailableShardsException(null, "at least one primary shard for the index [" + localState.concreteIndexName + "] is unavailable"); } else { @@ -195,16 +200,16 @@ public void clusterChanged(ClusterChangedEvent event) { } final State previousState = indexState; final IndexMetaData indexMetaData = resolveConcreteIndex(aliasName, event.state().metaData()); - final boolean indexExists = indexMetaData != null; - final boolean isIndexUpToDate = indexExists == false || + final Instant creationTime = indexMetaData != null ? Instant.ofEpochMilli(indexMetaData.getCreationDate()) : null; + final boolean isIndexUpToDate = indexMetaData == null || INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() == internalIndexFormat; final boolean indexAvailable = checkIndexAvailable(event.state()); - final boolean mappingIsUpToDate = indexExists == false || checkIndexMappingUpToDate(event.state()); + final boolean mappingIsUpToDate = indexMetaData == null || checkIndexMappingUpToDate(event.state()); final Version mappingVersion = oldestIndexMappingVersion(event.state()); final ClusterHealthStatus indexStatus = indexMetaData == null ? null : new ClusterIndexHealth(indexMetaData, event.state().getRoutingTable().index(indexMetaData.getIndex())).getStatus(); final String concreteIndexName = indexMetaData == null ? internalIndexName : indexMetaData.getIndex().getName(); - final State newState = new State(indexExists, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, concreteIndexName, + final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, concreteIndexName, indexStatus); this.indexState = newState; @@ -316,7 +321,7 @@ private static Version readMappingVersion(String indexName, MappingMetaData mapp */ public void checkIndexVersionThenExecute(final Consumer consumer, final Runnable andThen) { final State indexState = this.indexState; // use a local copy so all checks execute against the same state! - if (indexState.indexExists && indexState.isIndexUpToDate == false) { + if (indexState.indexExists() && indexState.isIndexUpToDate == false) { consumer.accept(new IllegalStateException( "Index [" + indexState.concreteIndexName + "] is not on the current version. Security features relying on the index" + " will not be available until the upgrade API is run on the index")); @@ -336,11 +341,11 @@ public void prepareIndexIfNeededThenExecute(final Consumer consumer, consumer.accept(new ElasticsearchStatusException( "Cluster state has not been recovered yet, cannot write to the [" + indexState.concreteIndexName + "] index", RestStatus.SERVICE_UNAVAILABLE)); - } else if (indexState.indexExists && indexState.isIndexUpToDate == false) { + } else if (indexState.indexExists() && indexState.isIndexUpToDate == false) { consumer.accept(new IllegalStateException( "Index [" + indexState.concreteIndexName + "] is not on the current version." + "Security features relying on the index will not be available until the upgrade API is run on the index")); - } else if (indexState.indexExists == false) { + } else if (indexState.indexExists() == false) { assert indexState.concreteIndexName != null; logger.info("security index does not exist. Creating [{}] with alias [{}]", indexState.concreteIndexName, this.aliasName); final byte[] mappingSource = mappingSourceSupplier.get(); @@ -435,8 +440,8 @@ private static Tuple parseMappingAndSettingsFromTemplateBytes( * State of the security index. */ public static class State { - public static final State UNRECOVERED_STATE = new State(false, false, false, false, null, null, null); - public final boolean indexExists; + public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null); + public final Instant creationTime; public final boolean isIndexUpToDate; public final boolean indexAvailable; public final boolean mappingUpToDate; @@ -444,9 +449,9 @@ public static class State { public final String concreteIndexName; public final ClusterHealthStatus indexStatus; - public State(boolean indexExists, boolean isIndexUpToDate, boolean indexAvailable, + public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailable, boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexStatus) { - this.indexExists = indexExists; + this.creationTime = creationTime; this.isIndexUpToDate = isIndexUpToDate; this.indexAvailable = indexAvailable; this.mappingUpToDate = mappingUpToDate; @@ -460,7 +465,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; State state = (State) o; - return indexExists == state.indexExists && + return creationTime == state.creationTime && isIndexUpToDate == state.isIndexUpToDate && indexAvailable == state.indexAvailable && mappingUpToDate == state.mappingUpToDate && @@ -469,9 +474,13 @@ public boolean equals(Object o) { indexStatus == state.indexStatus; } + public boolean indexExists() { + return creationTime != null; + } + @Override public int hashCode() { - return Objects.hash(indexExists, isIndexUpToDate, indexAvailable, mappingUpToDate, mappingVersion, concreteIndexName, + return Objects.hash(creationTime, isIndexUpToDate, indexAvailable, mappingUpToDate, mappingVersion, concreteIndexName, indexStatus); } } From de02138c083cf15b7e36c2be249df1504de1b39e Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 1 Apr 2019 17:33:57 +0300 Subject: [PATCH 06/50] Fix tests compilation --- .../xpack/security/authc/TokenService.java | 9 +- ...sportSamlInvalidateSessionActionTests.java | 2 +- .../saml/TransportSamlLogoutActionTests.java | 2 +- .../TransportCreateTokenActionTests.java | 6 +- .../authc/AuthenticationServiceTests.java | 4 +- .../security/authc/TokenServiceTests.java | 108 ++++++++++++------ .../authc/esnative/NativeRealmTests.java | 3 +- .../mapper/NativeRoleMappingStoreTests.java | 11 +- .../authz/store/CompositeRolesStoreTests.java | 11 +- 9 files changed, 101 insertions(+), 55 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 831853c5d1b85..ab76224b321d6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -180,6 +180,7 @@ public final class TokenService { private static final String TOKEN_DOC_ID_PREFIX = TOKEN_DOC_TYPE + "_"; static final int MINIMUM_BYTES = VERSION_BYTES + SALT_BYTES + IV_BYTES + 1; static final int MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * MINIMUM_BYTES) / 3)).intValue(); + static final Version VERSION_TOKENS_INDEX_INTRODUCED = Version.V_8_0_0; // TODO change upon backport private static final Logger logger = LogManager.getLogger(TokenService.class); private final SecureRandom secureRandom = new SecureRandom(); @@ -262,7 +263,7 @@ private void createOAuth2Tokens(String userTokenId, Version version, Authenticat final String plainRefreshToken = includeRefreshToken ? UUIDs.randomBase64UUID() : null; final String versionedRefreshToken = includeRefreshToken ? prependVersionAndEncode(version, plainRefreshToken) : null; final BytesReference tokenDocument; - if (version.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport + if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { tokenDocument = createTokenDocument(userToken, plainRefreshToken, originatingClientAuth); } else { tokenDocument = createTokenDocument(userToken, versionedRefreshToken, originatingClientAuth); @@ -569,7 +570,7 @@ private void indexInvalidation(Collection userTokens, Iterator idsOfRecentTokens = new HashSet<>(); final Set idsOfOlderTokens = new HashSet<>(); for (UserToken userToken : userTokens) { - if (userToken.getVersion().onOrAfter(Version.V_8_0_0)) { // change upon backport + if (userToken.getVersion().onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { idsOfRecentTokens.add(userToken.getId()); } else { idsOfOlderTokens.add(userToken.getId()); @@ -868,7 +869,7 @@ public void onFailure(Exception e) { final Map updateMap = new HashMap<>(); updateMap.put("refreshed", true); updateMap.put("refresh_time", clock.instant().toEpochMilli()); - if (newTokenVersion.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport + if (newTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { updateMap.put("superseded_by", prependVersionAndEncode(newTokenVersion, getTokenDocumentId(newUserTokenId))); } else { updateMap.put("superseded_by", getTokenDocumentId(newUserTokenId)); @@ -1317,7 +1318,7 @@ private void ensureEnabled() { * Therefore, in general, when searching for a token we need to consider both the new and the old indices. */ private SecurityIndexManager getSecurityIndexManagerForVersion(Version version) { - if (version.onOrAfter(Version.V_8_0_0)) { // TODO change upon backport + if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { return securityTokensIndex; } else { return securityMainIndex; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index de0463ffcaab3..7061a06fef614 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -198,7 +198,7 @@ void doExecute(Action action, Request request, ActionListener null, null, Collections.emptySet()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java index 22b9fb7a11127..bd1a20db2f196 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java @@ -203,7 +203,7 @@ public void setup() throws Exception { when(securityIndex.isAvailable()).thenReturn(true); final ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); - tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, clusterService); + tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, securityIndex, clusterService); final TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java index fb1399220474d..cea1a532e13c6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java @@ -147,7 +147,8 @@ public void stopThreadPool() throws Exception { } public void testClientCredentialsCreatesWithoutRefreshToken() throws Exception { - final TokenService tokenService = new TokenService(SETTINGS, Clock.systemUTC(), client, securityIndex, clusterService); + final TokenService tokenService = new TokenService(SETTINGS, Clock.systemUTC(), client, securityIndex, securityIndex, + clusterService); Authentication authentication = new Authentication(new User("joe"), new Authentication.RealmRef("realm", "type", "node"), null); authentication.writeToContext(threadPool.getThreadContext()); @@ -171,7 +172,8 @@ public void testClientCredentialsCreatesWithoutRefreshToken() throws Exception { } public void testPasswordGrantTypeCreatesWithRefreshToken() throws Exception { - final TokenService tokenService = new TokenService(SETTINGS, Clock.systemUTC(), client, securityIndex, clusterService); + final TokenService tokenService = new TokenService(SETTINGS, Clock.systemUTC(), client, securityIndex, securityIndex, + clusterService); Authentication authentication = new Authentication(new User("joe"), new Authentication.RealmRef("realm", "type", "node"), null); authentication.writeToContext(threadPool.getThreadContext()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index e05dd74d1275b..a86edb98251ff 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -220,7 +220,7 @@ licenseState, threadContext, mock(ReservedRealm.class), Arrays.asList(firstRealm }).when(securityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class)); ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex, clusterService, threadPool); - tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, clusterService); + tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, securityIndex, clusterService); service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, new AnonymousUser(settings), tokenService, apiKeyService); } @@ -1394,6 +1394,6 @@ private void setCompletedToTrue(AtomicBoolean completed) { } private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { - return new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, indexStatus); + return new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 3d050bd2af79c..3a1ec0f2ae948 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -21,6 +21,9 @@ import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -51,6 +54,7 @@ import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; import org.elasticsearch.xpack.security.support.SecurityIndexManager; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -74,6 +78,7 @@ import static java.time.Clock.systemUTC; import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; +import static org.elasticsearch.test.ClusterServiceUtils.setState; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -92,7 +97,8 @@ public class TokenServiceTests extends ESTestCase { .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true).build(); private Client client; - private SecurityIndexManager securityIndex; + private SecurityIndexManager securityMainIndex; + private SecurityIndexManager securityTokensIndex; private ClusterService clusterService; private Settings tokenServiceEnabledSettings = Settings.builder() .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true).build(); @@ -121,20 +127,44 @@ public void setupClient() { }).when(client).execute(eq(IndexAction.INSTANCE), any(IndexRequest.class), any(ActionListener.class)); // setup lifecycle service - securityIndex = mock(SecurityIndexManager.class); + this.securityMainIndex = mockSecurityManager(); + this.securityTokensIndex = mockSecurityManager(); + this.clusterService = ClusterServiceUtils.createClusterService(threadPool); + addAnotherDateNodeWithVersion(this.clusterService, Version.CURRENT); + } + + private SecurityIndexManager mockSecurityManager() { + SecurityIndexManager mockSecurityIndex = mock(SecurityIndexManager.class); doAnswer(invocationOnMock -> { Runnable runnable = (Runnable) invocationOnMock.getArguments()[1]; runnable.run(); return null; - }).when(securityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class)); + }).when(mockSecurityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class)); doAnswer(invocationOnMock -> { Runnable runnable = (Runnable) invocationOnMock.getArguments()[1]; runnable.run(); return null; - }).when(securityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class)); - when(securityIndex.indexExists()).thenReturn(true); - when(securityIndex.isAvailable()).thenReturn(true); - this.clusterService = ClusterServiceUtils.createClusterService(threadPool); + }).when(mockSecurityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class)); + when(mockSecurityIndex.indexExists()).thenReturn(true); + when(mockSecurityIndex.isAvailable()).thenReturn(true); + return mockSecurityIndex; + } + + private void addAnotherDateNodeWithVersion(ClusterService clusterService, Version version) { + final ClusterState currentState = clusterService.state(); + final DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(currentState.getNodes()); + final DiscoveryNode anotherDataNode = new DiscoveryNode("another_data_node#" + version, buildNewFakeTransportAddress(), + Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.DATA), version); + discoBuilder.add(anotherDataNode); + final ClusterState.Builder newStateBuilder = ClusterState.builder(currentState); + newStateBuilder.nodes(discoBuilder); + setState(clusterService, newStateBuilder.build()); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + clusterService.close(); } @BeforeClass @@ -151,7 +181,8 @@ public static void shutdownThreadpool() throws InterruptedException { } public void testAttachAndGetToken() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createOAuth2Tokens(authentication, authentication, Collections.emptyMap(), true, tokenFuture); @@ -172,8 +203,8 @@ public void testAttachAndGetToken() throws Exception { try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) { // verify a second separate token service with its own salt can also verify - TokenService anotherService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex - , clusterService); + TokenService anotherService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); anotherService.refreshMetaData(tokenService.getTokenMetaData()); PlainActionFuture future = new PlainActionFuture<>(); anotherService.getAndValidateToken(requestContext, future); @@ -183,7 +214,8 @@ public void testAttachAndGetToken() throws Exception { } public void testInvalidAuthorizationHeader() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); ThreadContext requestContext = new ThreadContext(Settings.EMPTY); String token = randomFrom("", " "); String authScheme = randomFrom("Bearer ", "BEARER ", "bearer ", "Basic "); @@ -198,7 +230,8 @@ public void testInvalidAuthorizationHeader() throws Exception { } public void testRotateKey() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createOAuth2Tokens(authentication, authentication, Collections.emptyMap(), true, tokenFuture); @@ -251,12 +284,14 @@ private void rotateKeys(TokenService tokenService) { } public void testKeyExchange() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); int numRotations = randomIntBetween(1, 5); for (int i = 0; i < numRotations; i++) { rotateKeys(tokenService); } - TokenService otherTokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService otherTokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); otherTokenService.refreshMetaData(tokenService.getTokenMetaData()); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); @@ -288,7 +323,8 @@ public void testKeyExchange() throws Exception { } public void testPruneKeys() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createOAuth2Tokens(authentication, authentication, Collections.emptyMap(), true, tokenFuture); @@ -350,7 +386,8 @@ public void testPruneKeys() throws Exception { } public void testPassphraseWorks() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createOAuth2Tokens(authentication, authentication, Collections.emptyMap(), true, tokenFuture); @@ -371,8 +408,8 @@ public void testPassphraseWorks() throws Exception { try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) { // verify a second separate token service with its own passphrase cannot verify - TokenService anotherService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, - clusterService); + TokenService anotherService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); PlainActionFuture future = new PlainActionFuture<>(); anotherService.getAndValidateToken(requestContext, future); assertNull(future.get()); @@ -380,7 +417,8 @@ public void testPassphraseWorks() throws Exception { } public void testGetTokenWhenKeyCacheHasExpired() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); @@ -393,9 +431,9 @@ public void testGetTokenWhenKeyCacheHasExpired() throws Exception { } public void testInvalidatedToken() throws Exception { - when(securityIndex.indexExists()).thenReturn(true); - TokenService tokenService = - new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + when(securityMainIndex.indexExists()).thenReturn(true); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createOAuth2Tokens(authentication, authentication, Collections.emptyMap(), true, tokenFuture); @@ -449,7 +487,8 @@ public void testTokenExpiryConfig() { public void testTokenExpiry() throws Exception { ClockMock clock = ClockMock.frozen(); - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, clock, client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, clock, client, securityMainIndex, securityTokensIndex, + clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createOAuth2Tokens(authentication, authentication, Collections.emptyMap(), true, tokenFuture); @@ -502,7 +541,7 @@ public void testTokenServiceDisabled() throws Exception { TokenService tokenService = new TokenService(Settings.builder() .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), false) .build(), - Clock.systemUTC(), client, securityIndex, clusterService); + Clock.systemUTC(), client, securityMainIndex, securityTokensIndex, clusterService); IllegalStateException e = expectThrows(IllegalStateException.class, () -> tokenService.createOAuth2Tokens(null, null, null, true, null)); assertEquals("tokens are not enabled", e.getMessage()); @@ -545,7 +584,8 @@ public void testMalformedToken() throws Exception { final int numBytes = randomIntBetween(1, TokenService.MINIMUM_BYTES + 32); final byte[] randomBytes = new byte[numBytes]; random().nextBytes(randomBytes); - TokenService tokenService = new TokenService(Settings.EMPTY, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(Settings.EMPTY, systemUTC(), client, securityMainIndex, securityTokensIndex, + clusterService); ThreadContext requestContext = new ThreadContext(Settings.EMPTY); requestContext.putHeader("Authorization", "Bearer " + Base64.getEncoder().encodeToString(randomBytes)); @@ -558,8 +598,8 @@ public void testMalformedToken() throws Exception { } public void testIndexNotAvailable() throws Exception { - TokenService tokenService = - new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createOAuth2Tokens(authentication, authentication, Collections.emptyMap(), true, tokenFuture); @@ -581,19 +621,19 @@ public void testIndexNotAvailable() throws Exception { tokenService.getAndValidateToken(requestContext, future); assertNull(future.get()); - when(securityIndex.isAvailable()).thenReturn(false); - when(securityIndex.indexExists()).thenReturn(true); + when(securityMainIndex.isAvailable()).thenReturn(false); + when(securityMainIndex.indexExists()).thenReturn(true); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); assertNull(future.get()); - when(securityIndex.indexExists()).thenReturn(false); + when(securityMainIndex.indexExists()).thenReturn(false); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); assertNull(future.get()); - when(securityIndex.isAvailable()).thenReturn(true); - when(securityIndex.indexExists()).thenReturn(true); + when(securityMainIndex.isAvailable()).thenReturn(true); + when(securityMainIndex.indexExists()).thenReturn(true); mockGetTokenFromId(token, false); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); @@ -602,8 +642,8 @@ public void testIndexNotAvailable() throws Exception { } public void testGetAuthenticationWorksWithExpiredUserToken() throws Exception { - TokenService tokenService = - new TokenService(tokenServiceEnabledSettings, Clock.systemUTC(), client, securityIndex, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, Clock.systemUTC(), client, securityMainIndex, + securityTokensIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); UserToken expired = new UserToken(authentication, Instant.now().minus(3L, ChronoUnit.DAYS)); mockGetTokenFromId(expired, false); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index c9313d20594ae..28625f20627e1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.security.support.SecurityIndexManager; +import java.time.Instant; import java.util.concurrent.atomic.AtomicInteger; import static org.mockito.Mockito.mock; @@ -26,7 +27,7 @@ public class NativeRealmTests extends ESTestCase { RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7); private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { - return new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, indexStatus); + return new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 4734d9248d820..8010787a710aa 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.hamcrest.Matchers; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -127,7 +128,7 @@ private String randomiseDn(String dn) { } private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { - return new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, indexStatus); + return new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus); } public void testCacheClearOnIndexHealthChange() { @@ -172,13 +173,13 @@ public void testCacheClearOnIndexOutOfDateChange() { final NativeRoleMappingStore store = buildRoleMappingStoreForInvalidationTesting(numInvalidation); store.onSecurityIndexStateChange( - new SecurityIndexManager.State(true, false, true, true, null, concreteSecurityIndexName, null), - new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, null)); + new SecurityIndexManager.State(Instant.now(), false, true, true, null, concreteSecurityIndexName, null), + new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, null)); assertEquals(1, numInvalidation.get()); store.onSecurityIndexStateChange( - new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, null), - new SecurityIndexManager.State(true, false, true, true, null, concreteSecurityIndexName, null)); + new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, null), + new SecurityIndexManager.State(Instant.now(), false, true, true, null, concreteSecurityIndexName, null)); assertEquals(2, numInvalidation.get()); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 90d9806655c35..9d83731575c61 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -58,6 +58,7 @@ import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -698,7 +699,7 @@ Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(Nativ } private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { - return new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, indexStatus); + return new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus); } public void testCacheClearOnIndexHealthChange() { @@ -773,13 +774,13 @@ public void invalidateAll() { }; compositeRolesStore.onSecurityIndexStateChange( - new SecurityIndexManager.State(true, false, true, true, null, concreteSecurityIndexName, null), - new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, null)); + new SecurityIndexManager.State(Instant.now(), false, true, true, null, concreteSecurityIndexName, null), + new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, null)); assertEquals(1, numInvalidation.get()); compositeRolesStore.onSecurityIndexStateChange( - new SecurityIndexManager.State(true, true, true, true, null, concreteSecurityIndexName, null), - new SecurityIndexManager.State(true, false, true, true, null, concreteSecurityIndexName, null)); + new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, null), + new SecurityIndexManager.State(Instant.now(), false, true, true, null, concreteSecurityIndexName, null)); assertEquals(2, numInvalidation.get()); } From f0e886a0515300efa9cf6d6bba7da8f504058ae7 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 1 Apr 2019 18:40:04 +0300 Subject: [PATCH 07/50] TokenServiceTests without anything new --- .../security/authc/TokenServiceTests.java | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 3a1ec0f2ae948..6f2ea942693ab 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.XPackSettings; @@ -130,35 +131,8 @@ public void setupClient() { this.securityMainIndex = mockSecurityManager(); this.securityTokensIndex = mockSecurityManager(); this.clusterService = ClusterServiceUtils.createClusterService(threadPool); - addAnotherDateNodeWithVersion(this.clusterService, Version.CURRENT); - } - - private SecurityIndexManager mockSecurityManager() { - SecurityIndexManager mockSecurityIndex = mock(SecurityIndexManager.class); - doAnswer(invocationOnMock -> { - Runnable runnable = (Runnable) invocationOnMock.getArguments()[1]; - runnable.run(); - return null; - }).when(mockSecurityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class)); - doAnswer(invocationOnMock -> { - Runnable runnable = (Runnable) invocationOnMock.getArguments()[1]; - runnable.run(); - return null; - }).when(mockSecurityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class)); - when(mockSecurityIndex.indexExists()).thenReturn(true); - when(mockSecurityIndex.isAvailable()).thenReturn(true); - return mockSecurityIndex; - } - - private void addAnotherDateNodeWithVersion(ClusterService clusterService, Version version) { - final ClusterState currentState = clusterService.state(); - final DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(currentState.getNodes()); - final DiscoveryNode anotherDataNode = new DiscoveryNode("another_data_node#" + version, buildNewFakeTransportAddress(), - Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.DATA), version); - discoBuilder.add(anotherDataNode); - final ClusterState.Builder newStateBuilder = ClusterState.builder(currentState); - newStateBuilder.nodes(discoBuilder); - setState(clusterService, newStateBuilder.build()); + addAnotherDateNodeWithVersion(this.clusterService, + randomFrom(VersionUtils.getPreviousVersion(TokenService.VERSION_TOKENS_INDEX_INTRODUCED), Version.CURRENT)); } @After @@ -712,4 +686,32 @@ protected String getDeprecatedAccessTokenString(TokenService tokenService, UserT } } + private SecurityIndexManager mockSecurityManager() { + SecurityIndexManager mockSecurityIndex = mock(SecurityIndexManager.class); + doAnswer(invocationOnMock -> { + Runnable runnable = (Runnable) invocationOnMock.getArguments()[1]; + runnable.run(); + return null; + }).when(mockSecurityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class)); + doAnswer(invocationOnMock -> { + Runnable runnable = (Runnable) invocationOnMock.getArguments()[1]; + runnable.run(); + return null; + }).when(mockSecurityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class)); + when(mockSecurityIndex.indexExists()).thenReturn(true); + when(mockSecurityIndex.isAvailable()).thenReturn(true); + return mockSecurityIndex; + } + + private void addAnotherDateNodeWithVersion(ClusterService clusterService, Version version) { + final ClusterState currentState = clusterService.state(); + final DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(currentState.getNodes()); + final DiscoveryNode anotherDataNode = new DiscoveryNode("another_data_node#" + version, buildNewFakeTransportAddress(), + Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.DATA), version); + discoBuilder.add(anotherDataNode); + final ClusterState.Builder newStateBuilder = ClusterState.builder(currentState); + newStateBuilder.nodes(discoBuilder); + setState(clusterService, newStateBuilder.build()); + } + } From 56e5832a6e2541d4e524d82aa64c6e88920c956f Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 1 Apr 2019 19:02:43 +0300 Subject: [PATCH 08/50] mistakes --- .../src/main/resources/security-tokens-index-template-7.json | 2 +- .../security/authc/support/mapper/NativeRoleMappingStore.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json index 28edfa847f75d..cef76c8975221 100644 --- a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json +++ b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json @@ -6,7 +6,7 @@ "number_of_replicas" : 0, "auto_expand_replicas" : "0-1", "index.priority": 1000, - "index.format": 7, + "index.format": 7 }, "mappings" : { "_doc" : { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index 08bbcc1046f4d..78932c900320e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel; import org.elasticsearch.xpack.core.security.client.SecurityClient; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.security.authc.support.CachingRealm; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.security.support.SecurityIndexManager; @@ -67,7 +68,7 @@ /** * This store reads + writes {@link ExpressionRoleMapping role mappings} in an Elasticsearch - * {@link SecurityIndexManager#SECURITY_MAIN_ALIAS_NAME index}. + * {@link RestrictedIndicesNames#SECURITY_MAIN_ALIAS index}. *
* The store is responsible for all read and write operations as well as * {@link #resolveRoles(UserData, ActionListener) resolving roles}. From 90b51c55366b69b15c01216cc850d1566368f4de Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 1 Apr 2019 20:39:05 +0300 Subject: [PATCH 09/50] versionCompatibilityOverride --- .../xpack/security/authc/TokenService.java | 17 ++++++++++++++--- .../security/authc/TokenAuthIntegTests.java | 16 ++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index ab76224b321d6..065dba8b24cc6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -130,6 +130,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -197,6 +198,8 @@ public final class TokenService { private volatile TokenKeys keyCache; private volatile long lastExpirationRunMs; private final AtomicLong createdTimeStamps = new AtomicLong(-1); + // for testing only + private final AtomicReference versionCompatibilityOverride = new AtomicReference<>(); /** * Creates a new token service @@ -225,6 +228,14 @@ public TokenService(Settings settings, Clock clock, Client client, SecurityIndex getTokenMetaData(); } + private Version getVersionCompatibility() { + final Version versionOverride = versionCompatibilityOverride.get(); + if (versionOverride != null) { + return versionOverride; + } + return clusterService.state().nodes().getMinNodeVersion(); + } + public static Boolean isTokenServiceEnabled(Settings settings) { return XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get(settings); } @@ -237,10 +248,10 @@ public void createOAuth2Tokens(Authentication authentication, Authentication ori Map metadata, boolean includeRefreshToken, ActionListener> listener) { // the created token is compatible with the oldest node version in the cluster - final Version minNodeVersion = clusterService.state().nodes().getMinNodeVersion(); + final Version tokenVersion = getVersionCompatibility(); // the id of the created tokens ought be unguessable final String userTokenId = UUIDs.randomBase64UUID(); - createOAuth2Tokens(userTokenId, minNodeVersion, authentication, originatingClientAuth, metadata, includeRefreshToken, listener); + createOAuth2Tokens(userTokenId, tokenVersion, authentication, originatingClientAuth, metadata, includeRefreshToken, listener); } /** @@ -865,7 +876,7 @@ public void onFailure(Exception e) { }); } else { final String newUserTokenId = UUIDs.randomBase64UUID(); - final Version newTokenVersion = clusterService.state().nodes().getMinNodeVersion(); + final Version newTokenVersion = getVersionCompatibility(); final Map updateMap = new HashMap<>(); updateMap.put("refreshed", true); updateMap.put("refresh_time", clock.instant().toEpochMilli()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index c52896660a650..7bac18cfcfb67 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.core.security.authc.TokenMetaData; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; +import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.junit.After; import org.junit.Before; @@ -54,7 +55,6 @@ import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; -import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.equalTo; @TestLogging("org.elasticsearch.xpack.security.authz.store.FileRolesStore:DEBUG") @@ -160,7 +160,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0)); AtomicReference docId = new AtomicReference<>(); assertBusy(() -> { - SearchResponse searchResponse = client.prepareSearch(SECURITY_MAIN_ALIAS) + SearchResponse searchResponse = client.prepareSearch(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.termQuery("doc_type", "token"))) .setSize(1) @@ -173,7 +173,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { // hack doc to modify the creation time to the day before Instant yesterday = created.minus(36L, ChronoUnit.HOURS); assertTrue(Instant.now().isAfter(yesterday)); - client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, docId.get()) + client.prepareUpdate(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, SINGLE_MAPPING_NAME, docId.get()) .setDoc("creation_time", yesterday.toEpochMilli()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -191,8 +191,8 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { assertEquals("token malformed", e.getMessage()); } } - client.admin().indices().prepareRefresh(SECURITY_MAIN_ALIAS).get(); - SearchResponse searchResponse = client.prepareSearch(SECURITY_MAIN_ALIAS) + client.admin().indices().prepareRefresh(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS).get(); + SearchResponse searchResponse = client.prepareSearch(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.termQuery("doc_type", "token"))) .setTerminateAfter(1) @@ -357,10 +357,10 @@ public void testRefreshingMultipleTimesFails() throws Exception { // We now have two documents, the original(now refreshed) token doc and the new one with the new access doc AtomicReference docId = new AtomicReference<>(); assertBusy(() -> { - SearchResponse searchResponse = client.prepareSearch(SECURITY_MAIN_ALIAS) + SearchResponse searchResponse = client.prepareSearch(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("doc_type", "token")) + .must(QueryBuilders.termQuery("doc_type", TokenService.TOKEN_DOC_TYPE)) .must(QueryBuilders.termQuery("refresh_token.refreshed", "true")))) .setSize(1) .setTerminateAfter(1) @@ -373,7 +373,7 @@ public void testRefreshingMultipleTimesFails() throws Exception { Instant refreshed = Instant.now(); Instant aWhileAgo = refreshed.minus(50L, ChronoUnit.SECONDS); assertTrue(Instant.now().isAfter(aWhileAgo)); - UpdateResponse updateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, docId.get()) + UpdateResponse updateResponse = client.prepareUpdate(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, SINGLE_MAPPING_NAME, docId.get()) .setDoc("refresh_token", Collections.singletonMap("refresh_time", aWhileAgo.toEpochMilli())) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .setFetchSource("refresh_token", Strings.EMPTY_STRING) From 68ccfd47ba6ed063e7935661894e98754a9608cc Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 1 Apr 2019 21:11:30 +0300 Subject: [PATCH 10/50] Invalidation carry-over --- .../xpack/security/authc/TokenService.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 065dba8b24cc6..2ae893c59cfcd 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -588,16 +588,16 @@ private void indexInvalidation(Collection userTokens, Iterator { + indexInvalidation(idsOfOlderTokens, securityMainIndex, backoff, srcPrefix, previousResult, ActionListener.wrap(newResult -> { if (false == idsOfRecentTokens.isEmpty()) { - // carry-over result of the invalidation for the main security index - indexInvalidation(idsOfRecentTokens, securityTokensIndex, backoff, srcPrefix, result, listener); + // carry-over result of the invalidation for the tokens security index + indexInvalidation(idsOfRecentTokens, securityTokensIndex, backoff, srcPrefix, newResult, listener); } else { - listener.onResponse(result); + listener.onResponse(newResult); } }, listener::onFailure)); } else { - indexInvalidation(idsOfRecentTokens, securityTokensIndex, backoff, srcPrefix, null, listener); + indexInvalidation(idsOfRecentTokens, securityTokensIndex, backoff, srcPrefix, previousResult, listener); } } From 8e0c518419c73fc01d8d66348b3657dc19cde6e4 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 1 Apr 2019 23:59:13 +0300 Subject: [PATCH 11/50] TokenServiceAuth --- .../elasticsearch/xpack/security/authc/TokenService.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 2ae893c59cfcd..d6cfd47ee1daf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -1291,13 +1291,16 @@ private Tuple filterAndParseHit(SearchHit hit, @Nullable Pred */ private Tuple parseTokensFromDocument(Map source, @Nullable Predicate> filter) throws IllegalStateException, DateTimeException { - final String refreshToken = (String) ((Map) source.get("refresh_token")).get("token"); + final String plainRefreshToken = (String) ((Map) source.get("refresh_token")).get("token"); final Map userTokenSource = (Map) ((Map) source.get("access_token")).get("user_token"); if (null != filter && filter.test(userTokenSource) == false) { return null; } - return new Tuple<>(UserToken.fromSourceMap(userTokenSource), refreshToken); + final UserToken userToken = UserToken.fromSourceMap(userTokenSource); + final String versionedRefreshToken = plainRefreshToken != null ? + prependVersionAndEncode(userToken.getVersion(), plainRefreshToken) : null; + return new Tuple<>(userToken, versionedRefreshToken); } private static String getTokenDocumentId(UserToken userToken) { From 7842104e0b763ff946a0b4a5791fe62026546ed7 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 2 Apr 2019 16:14:54 +0300 Subject: [PATCH 12/50] Old tests finally working --- .../xpack/security/authc/TokenService.java | 47 +++++++++-------- .../support/SecurityIndexManager.java | 2 +- .../security/authc/TokenServiceTests.java | 33 ++++++++---- .../support/SecurityIndexManagerTests.java | 51 ++++++++++--------- 4 files changed, 75 insertions(+), 58 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index d6cfd47ee1daf..09144803e6db3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -182,6 +182,8 @@ public final class TokenService { static final int MINIMUM_BYTES = VERSION_BYTES + SALT_BYTES + IV_BYTES + 1; static final int MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * MINIMUM_BYTES) / 3)).intValue(); static final Version VERSION_TOKENS_INDEX_INTRODUCED = Version.V_8_0_0; // TODO change upon backport + static final Version VERSION_ACCESS_TOKENS_AS_UUIDS = Version.V_7_1_0; + static final Version VERSION_MULTIPLE_CONCURRENT_REFRESHES = Version.V_7_1_0; private static final Logger logger = LogManager.getLogger(TokenService.class); private final SecureRandom secureRandom = new SecureRandom(); @@ -391,24 +393,24 @@ void getUserTokenFromId(String userTokenId, Version version, ActionListener listener) { final byte[] bytes = token.getBytes(StandardCharsets.UTF_8); try (StreamInput in = new InputStreamStreamInput(Base64.getDecoder().wrap(new ByteArrayInputStream(bytes)), bytes.length)) { final Version version = Version.readVersion(in); in.setVersion(version); - if (version.onOrAfter(Version.V_7_1_0)) { - // The token was created in a > 7.1.0 cluster so it contains the tokenId as a String + if (version.onOrAfter(VERSION_ACCESS_TOKENS_AS_UUIDS)) { + // The token was created in a > VERSION_ACCESS_TOKENS_UUIDS cluster so it contains the tokenId as a String String usedTokenId = in.readString(); getUserTokenFromId(usedTokenId, version, listener); } else { - // The token was created in a < 7.1.0 cluster so we need to decrypt it to get the tokenId + // The token was created in a < VERSION_ACCESS_TOKENS_UUIDS cluster so we need to decrypt it to get the tokenId if (in.available() < MINIMUM_BASE64_BYTES) { logger.debug("invalid token, smaller than [{}] bytes", MINIMUM_BASE64_BYTES); listener.onResponse(null); @@ -1071,13 +1073,13 @@ private static Map getUserTokenSourceMap(Map sou * @return An {@code Optional} containing the exception in case this refresh token cannot be reused, or an empty Optional if * refreshing is allowed. */ - private static Optional checkMultipleRefreshes(Instant now, RefreshTokenStatus refreshToken) { - if (refreshToken.isRefreshed()) { - if (refreshToken.getVersion().onOrAfter(Version.V_7_1_0)) { - if (now.isAfter(refreshToken.getRefreshInstant().plus(30L, ChronoUnit.SECONDS))) { + private static Optional checkMultipleRefreshes(Instant now, RefreshTokenStatus refreshTokenStatus) { + if (refreshTokenStatus.isRefreshed()) { + if (refreshTokenStatus.getVersion().onOrAfter(VERSION_MULTIPLE_CONCURRENT_REFRESHES)) { + if (now.isAfter(refreshTokenStatus.getRefreshInstant().plus(30L, ChronoUnit.SECONDS))) { return Optional.of(invalidGrantException("token has already been refreshed more than 30 seconds in the past")); } - if (now.isBefore(refreshToken.getRefreshInstant().minus(30L, ChronoUnit.SECONDS))) { + if (now.isBefore(refreshTokenStatus.getRefreshInstant().minus(30L, ChronoUnit.SECONDS))) { return Optional .of(invalidGrantException("token has been refreshed more than 30 seconds in the future, clock skew too great")); } @@ -1326,10 +1328,11 @@ private void ensureEnabled() { } /** - * In version 7.1 tokens moved into a separate index away from the other entities due to their ephemeral nature. But they moved - * "seamlessly" - without manual intervention. In this way, new tokens are created in the new index, while the existing ones were left - * in place - to be accessed from the old index - and due to be removed automatically by the {@code ExpiredTokenRemover} periodic job. - * Therefore, in general, when searching for a token we need to consider both the new and the old indices. + * In version {@code #VERSION_TOKENS_INDEX_INTRODUCED} tokens moved into a separate index away from the other entities due to their + * ephemeral nature. But they moved "seamlessly" - without manual intervention. In this way, new tokens are created in the new index, + * while the existing ones were left in place - to be accessed from the old index - and due to be removed automatically by the {@code + * ExpiredTokenRemover} periodic job. Therefore, in general, when searching for a token we need to consider both the new and the old + * indices. */ private SecurityIndexManager getSecurityIndexManagerForVersion(Version version) { if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { @@ -1425,12 +1428,12 @@ private String getFromHeader(ThreadContext threadContext) { } /** - * Serializes a token to a String containing the version of the node that created the token and - * either an encrypted representation of the token id for versions earlier to 7.1.0 or the token ie - * itself for versions after 7.1.0 + * Serializes a token to a String containing the minimum compatible node version for decoding it back and either an encrypted + * representation of the token id for versions earlier to {@code #VERSION_ACCESS_TOKENS_UUIDS} the token ie itself for versions after + * {@code #VERSION_ACCESS_TOKENS_UUIDS} */ public String getAccessTokenAsString(UserToken userToken) throws IOException, GeneralSecurityException { - if (clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_7_1_0)) { + if (userToken.getVersion().onOrAfter(VERSION_ACCESS_TOKENS_AS_UUIDS)) { try (ByteArrayOutputStream os = new ByteArrayOutputStream(MINIMUM_BASE64_BYTES); OutputStream base64 = Base64.getEncoder().wrap(os); StreamOutput out = new OutputStreamStreamOutput(base64)) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 254527e5f71f9..5dbed1a9af730 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -465,7 +465,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; State state = (State) o; - return creationTime == state.creationTime && + return Objects.equals(creationTime, state.creationTime) && isIndexUpToDate == state.isIndexUpToDate && indexAvailable == state.indexAvailable && mappingUpToDate == state.mappingUpToDate && diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 6f2ea942693ab..4ac877007ff28 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; -import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.XPackSettings; @@ -101,6 +100,7 @@ public class TokenServiceTests extends ESTestCase { private SecurityIndexManager securityMainIndex; private SecurityIndexManager securityTokensIndex; private ClusterService clusterService; + private DiscoveryNode oldNode; private Settings tokenServiceEnabledSettings = Settings.builder() .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true).build(); @@ -131,8 +131,12 @@ public void setupClient() { this.securityMainIndex = mockSecurityManager(); this.securityTokensIndex = mockSecurityManager(); this.clusterService = ClusterServiceUtils.createClusterService(threadPool); - addAnotherDateNodeWithVersion(this.clusterService, - randomFrom(VersionUtils.getPreviousVersion(TokenService.VERSION_TOKENS_INDEX_INTRODUCED), Version.CURRENT)); + // version 7.1 was an "inflection" point in the Token Service development (access_tokens as UUIDS, multiple concurrent refreshes, + // tokens docs on a separate index), let's test the TokenService works in a mixed cluster with nodes with versions prior to these + // developments + if (randomBoolean()) { + oldNode = addAnotherDateNodeWithVersion(this.clusterService, randomFrom(Version.V_6_7_0, Version.V_7_0_0)); + } } @After @@ -281,7 +285,7 @@ public void testKeyExchange() throws Exception { PlainActionFuture future = new PlainActionFuture<>(); otherTokenService.getAndValidateToken(requestContext, future); UserToken serialized = future.get(); - assertAuthentication(authentication, serialized.getAuthentication()); + assertEquals(authentication, serialized.getAuthentication()); } rotateKeys(tokenService); @@ -590,28 +594,34 @@ public void testIndexNotAvailable() throws Exception { return Void.TYPE; }).when(client).get(any(GetRequest.class), any(ActionListener.class)); + final SecurityIndexManager tokensIndex; + if (oldNode != null) { + tokensIndex = securityMainIndex; + } else { + tokensIndex = securityTokensIndex; + } try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) { PlainActionFuture future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); assertNull(future.get()); - when(securityMainIndex.isAvailable()).thenReturn(false); - when(securityMainIndex.indexExists()).thenReturn(true); + when(tokensIndex.isAvailable()).thenReturn(false); + when(tokensIndex.indexExists()).thenReturn(true); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); assertNull(future.get()); - when(securityMainIndex.indexExists()).thenReturn(false); + when(tokensIndex.indexExists()).thenReturn(false); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); assertNull(future.get()); - when(securityMainIndex.isAvailable()).thenReturn(true); - when(securityMainIndex.indexExists()).thenReturn(true); + when(tokensIndex.isAvailable()).thenReturn(true); + when(tokensIndex.indexExists()).thenReturn(true); mockGetTokenFromId(token, false); future = new PlainActionFuture<>(); tokenService.getAndValidateToken(requestContext, future); - assertEquals(token.getAuthentication(), future.get().getAuthentication()); + assertEquals(future.get().getAuthentication(), token.getAuthentication()); } } @@ -703,7 +713,7 @@ private SecurityIndexManager mockSecurityManager() { return mockSecurityIndex; } - private void addAnotherDateNodeWithVersion(ClusterService clusterService, Version version) { + private DiscoveryNode addAnotherDateNodeWithVersion(ClusterService clusterService, Version version) { final ClusterState currentState = clusterService.state(); final DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(currentState.getNodes()); final DiscoveryNode anotherDataNode = new DiscoveryNode("another_data_node#" + version, buildNewFakeTransportAddress(), @@ -712,6 +722,7 @@ private void addAnotherDateNodeWithVersion(ClusterService clusterService, Versio final ClusterState.Builder newStateBuilder = ClusterState.builder(currentState); newStateBuilder.nodes(discoBuilder); setState(clusterService, newStateBuilder.build()); + return anotherDataNode; } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 4b6581d41a9eb..2337efdb4fb3a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -101,7 +102,8 @@ void doExecute(Action action, Request request, ActionListener { @@ -255,8 +258,8 @@ public void testListeneredNotCalledBeforeStateNotRecovered() throws Exception { assertThat(manager.isStateRecovered(), is(false)); assertThat(listenerCalled.get(), is(false)); // state recovered with index - ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, TEMPLATE_NAME, - SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); + ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertThat(manager.isStateRecovered(), is(true)); @@ -278,8 +281,8 @@ public void testIndexOutOfDateListeners() throws Exception { assertTrue(manager.isIndexUpToDate()); // index doesn't exist and now exists with wrong format - ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, TEMPLATE_NAME, - SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT - 1); + ClusterState.Builder clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT - 1); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); @@ -295,8 +298,8 @@ public void testIndexOutOfDateListeners() throws Exception { listenerCalled.set(false); // index doesn't exist and now exists with correct format - clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, TEMPLATE_NAME, - SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); + clusterStateBuilder = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); @@ -318,18 +321,18 @@ private void assertIndexUpToDateButNotAvailable() { assertThat(manager.isStateRecovered(), Matchers.equalTo(true)); } - public static ClusterState.Builder createClusterState(String indexName, String templateName) throws IOException { - return createClusterState(indexName, templateName, templateName, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); + public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName) throws IOException { + return createClusterState(indexName, aliasName, templateName, templateName, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); } - public static ClusterState.Builder createClusterState(String indexName, String templateName, int format) throws IOException { - return createClusterState(indexName, templateName, templateName, format); + public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, int format) throws IOException { + return createClusterState(indexName, aliasName, templateName, templateName, format); } - private static ClusterState.Builder createClusterState(String indexName, String templateName, String buildMappingFrom, int format) + private static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, String buildMappingFrom, int format) throws IOException { IndexTemplateMetaData.Builder templateBuilder = getIndexTemplateMetaData(templateName); - IndexMetaData.Builder indexMeta = getIndexMetadata(indexName, buildMappingFrom, format); + IndexMetaData.Builder indexMeta = getIndexMetadata(indexName, aliasName, buildMappingFrom, format); MetaData.Builder metaDataBuilder = new MetaData.Builder(); metaDataBuilder.put(templateBuilder); @@ -350,7 +353,7 @@ private static ClusterState state() { .build(); } - private static IndexMetaData.Builder getIndexMetadata(String indexName, String templateName, int format) throws IOException { + private static IndexMetaData.Builder getIndexMetadata(String indexName, String aliasName, String templateName, int format) throws IOException { IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName); indexMetaData.settings(Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) @@ -358,7 +361,7 @@ private static IndexMetaData.Builder getIndexMetadata(String indexName, String t .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), format) .build()); - + indexMetaData.putAlias(AliasMetaData.builder(aliasName).build()); final Map mappings = getTemplateMappings(templateName); for (Map.Entry entry : mappings.entrySet()) { indexMetaData.putMapping(entry.getKey(), entry.getValue()); @@ -398,13 +401,13 @@ public void testMappingVersionMatching() throws IOException { } public void testMissingVersionMappingThrowsError() throws IOException { - String templateString = "/missing-version-" + SECURITY_MAIN_TEMPLATE_7 + ".json"; + String templateString = "/missing-version-security-index-template.json"; ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); final ClusterState clusterState = clusterStateBuilder.build(); IllegalStateException exception = expectThrows(IllegalStateException.class, - () -> SecurityIndexManager.checkIndexMappingVersionMatches(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + () -> SecurityIndexManager.checkIndexMappingVersionMatches(RestrictedIndicesNames.SECURITY_MAIN_ALIAS, clusterState, logger, Version.CURRENT::equals)); - assertEquals("Cannot read security-version string in index " + RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, + assertEquals("Cannot read security-version string in index " + RestrictedIndicesNames.SECURITY_MAIN_ALIAS, exception.getMessage()); } @@ -462,7 +465,7 @@ private ClusterState.Builder createClusterStateWithTemplate(String securityTempl private ClusterState.Builder createClusterStateWithMapping(String securityTemplateString) throws IOException { final ClusterState clusterState = createClusterStateWithIndex(securityTemplateString).build(); final String indexName = clusterState.metaData().getAliasAndIndexLookup() - .get(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7).getIndices().get(0).getIndex().getName(); + .get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).getIndices().get(0).getIndex().getName(); return ClusterState.builder(clusterState).routingTable(SecurityTestUtils.buildIndexRoutingTable(indexName)); } @@ -496,7 +499,7 @@ private static IndexMetaData.Builder createIndexMetadata(String indexName, Strin private ClusterState.Builder createClusterStateWithIndex(String securityTemplate) throws IOException { final MetaData.Builder metaDataBuilder = new MetaData.Builder(); final boolean withAlias = randomBoolean(); - final String securityIndexName = RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7 + final String securityIndexName = RestrictedIndicesNames.SECURITY_MAIN_ALIAS + (withAlias ? "-" + randomAlphaOfLength(5) : ""); metaDataBuilder.put(createIndexMetadata(securityIndexName, securityTemplate)); From 2ef05814be8eb01b3487cdc0d2a151e9a4a17240 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 2 Apr 2019 17:29:54 +0300 Subject: [PATCH 13/50] Checkstyle --- .../src/main/java/org/elasticsearch/test/ESTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 6b36f985c210b..cd1a9cf24cf8e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -47,7 +47,6 @@ import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TimeUnits; import org.elasticsearch.Version; -import org.elasticsearch.bootstrap.BootstrapForTesting; import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterModule; From f88614244fd17ddc4ecadaf94878dd9ddcfcb8ae Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 3 Apr 2019 12:34:48 +0300 Subject: [PATCH 14/50] Checkstyle --- .../src/main/java/org/elasticsearch/test/ESTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index cd1a9cf24cf8e..6b36f985c210b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -47,6 +47,7 @@ import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TimeUnits; import org.elasticsearch.Version; +import org.elasticsearch.bootstrap.BootstrapForTesting; import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterModule; From 07eb32ffc911825f67d6c82b527d216a47569d49 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 3 Apr 2019 13:39:02 +0300 Subject: [PATCH 15/50] Checkstyle --- .../xpack/security/authc/TokenService.java | 7 ++++--- .../xpack/security/support/SecurityIndexManager.java | 4 ++-- .../xpack/security/authc/ApiKeyIntegTests.java | 1 - .../authc/esnative/ESNativeMigrateToolTests.java | 1 - .../security/authc/esnative/NativeRealmIntegTests.java | 4 ++-- .../authz/store/NativePrivilegeStoreTests.java | 4 ++-- .../security/support/SecurityIndexManagerTests.java | 10 ++++++---- .../xpack/restart/FullClusterRestartIT.java | 1 - 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 09144803e6db3..eca0d1e922db1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -741,8 +741,8 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator /** * Performs an asynchronous search request for the token document that contains the {@code refreshToken} and calls the listener with the - * {@link SearchResponse}. In case of recoverable errors the {@code SearchRequest} is retried using an exponential backoff policy. This method - * requires the tokens index where the token document, pointed to by the refresh token, resides. + * {@link SearchResponse}. In case of recoverable errors the {@code SearchRequest} is retried using an exponential backoff policy. This + * method requires the tokens index where the token document, pointed to by the refresh token, resides. */ private void findTokenFromRefreshToken(String refreshToken, SecurityIndexManager tokensIndexManager, Iterator backoff, ActionListener listener) { @@ -890,7 +890,8 @@ public void onFailure(Exception e) { assert seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; assert primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM : "expected an assigned primary term"; final SecurityIndexManager refreshedTokenIndex = getSecurityIndexManagerForVersion(refreshTokenStatus.getVersion()); - final UpdateRequestBuilder updateRequest = client.prepareUpdate(refreshedTokenIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) + final UpdateRequestBuilder updateRequest = client + .prepareUpdate(refreshedTokenIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) .setDoc("refresh_token", updateMap) .setFetchSource(true) .setRefreshPolicy(RefreshPolicy.IMMEDIATE) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 5dbed1a9af730..12a890dcbe6cd 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -209,8 +209,8 @@ public void clusterChanged(ClusterChangedEvent event) { final ClusterHealthStatus indexStatus = indexMetaData == null ? null : new ClusterIndexHealth(indexMetaData, event.state().getRoutingTable().index(indexMetaData.getIndex())).getStatus(); final String concreteIndexName = indexMetaData == null ? internalIndexName : indexMetaData.getIndex().getName(); - final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, concreteIndexName, - indexStatus); + final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, + concreteIndexName, indexStatus); this.indexState = newState; if (newState.equals(previousState) == false) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 071136d8608bf..f6849cae4c1cd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.junit.After; import org.junit.Before; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java index 9cd95777055f4..a73fc93f32e45 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.CharArrays; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.BeforeClass; import java.nio.charset.StandardCharsets; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java index 0c109b981ef7c..9e7371f95ed8c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java @@ -52,7 +52,6 @@ import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.Before; import org.junit.BeforeClass; @@ -596,7 +595,8 @@ public void testUsersAndRolesDoNotInterfereWithIndicesStats() throws Exception { assertThat(response.getFailedShards(), is(0)); assertThat(response.getIndices().size(), is(2)); assertThat(response.getIndices().get(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7), notNullValue()); - assertThat(response.getIndices().get(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7).getIndex(), is(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); + assertThat(response.getIndices().get(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7).getIndex(), + is(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); } public void testOperationsOnReservedUsers() throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index c58c1eb6d5c71..95ae9c5351806 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -188,7 +188,7 @@ public void testGetPrivilegesByWildcardApplicationName() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(SearchRequest.class)); SearchRequest request = (SearchRequest) requests.get(0); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); final String query = Strings.toString(request.source().query()); assertThat(query, containsString("{\"bool\":{\"filter\":[{\"terms\":{\"application\":[\"yourapp\"]")); @@ -208,7 +208,7 @@ public void testGetPrivilegesByStarApplicationName() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(SearchRequest.class)); SearchRequest request = (SearchRequest) requests.get(0); - assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME)); + assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); final String query = Strings.toString(request.source().query()); assertThat(query, containsString("{\"exists\":{\"field\":\"application\"")); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 2337efdb4fb3a..3dd5395b1fea0 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -325,12 +325,13 @@ public static ClusterState.Builder createClusterState(String indexName, String a return createClusterState(indexName, aliasName, templateName, templateName, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT); } - public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, int format) throws IOException { + public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, int format) + throws IOException { return createClusterState(indexName, aliasName, templateName, templateName, format); } - private static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, String buildMappingFrom, int format) - throws IOException { + private static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, String buildMappingFrom, + int format) throws IOException { IndexTemplateMetaData.Builder templateBuilder = getIndexTemplateMetaData(templateName); IndexMetaData.Builder indexMeta = getIndexMetadata(indexName, aliasName, buildMappingFrom, format); @@ -353,7 +354,8 @@ private static ClusterState state() { .build(); } - private static IndexMetaData.Builder getIndexMetadata(String indexName, String aliasName, String templateName, int format) throws IOException { + private static IndexMetaData.Builder getIndexMetadata(String indexName, String aliasName, String templateName, int format) + throws IOException { IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName); indexMetaData.settings(Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index a670a77281cb3..41c08d5919ba9 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.upgrade.UpgradeField; import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.watcher.actions.index.IndexAction; import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; From 87792e4840d8436d70e66fc7c6748a7be01f2f82 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 3 Apr 2019 19:35:40 +0300 Subject: [PATCH 16/50] Nits in Tim's review --- .../org/elasticsearch/xpack/security/authc/TokenService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index eca0d1e922db1..20b0d139f0a17 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -1126,6 +1126,7 @@ public void findActiveTokensForRealm(String realmName, @Nullable Predicate> filter, ActionListener>> listener) { @@ -1231,7 +1232,8 @@ private void findActiveTokensForUser(String username, SecurityIndexManager token private BytesReference createTokenDocument(UserToken userToken, @Nullable String refreshToken, @Nullable Authentication originatingClientAuth) { - assert refreshToken == null || originatingClientAuth != null; + assert refreshToken == null || originatingClientAuth != null : "non-null refresh token " + refreshToken + + " requires non-null client authn " + originatingClientAuth; try (XContentBuilder builder = XContentFactory.jsonBuilder()) { builder.startObject(); builder.field("doc_type", TOKEN_DOC_TYPE); @@ -1349,6 +1351,7 @@ private SecurityIndexManager getSecurityIndexManagerForVersion(Version version) private void checkIfTokenIsValid(UserToken userToken, ActionListener listener) { if (clock.instant().isAfter(userToken.getExpirationTime())) { listener.onFailure(traceLog("validate token", userToken.getId(), expiredTokenException())); + return; } final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(userToken.getVersion()); if (tokensIndex.indexExists() == false) { From 21b48fb6670c7f81a4887af357503e8d0b68f1da Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 3 Apr 2019 20:10:04 +0300 Subject: [PATCH 17/50] findTokenFromRefreshToken bug --- .../xpack/security/authc/TokenService.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 20b0d139f0a17..1827e0b6acee4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -282,7 +282,7 @@ private void createOAuth2Tokens(String userTokenId, Version version, Authenticat tokenDocument = createTokenDocument(userToken, versionedRefreshToken, originatingClientAuth); } final String documentId = getTokenDocumentId(userToken); - final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(version); + final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(version); final IndexRequest indexTokenRequest = client.prepareIndex(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) .setOpType(OpType.CREATE) .setSource(tokenDocument, XContentType.JSON) @@ -346,7 +346,7 @@ public void getAuthenticationAndMetaData(String token, ActionListener listener) { - final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(version); + final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(version); if (tokensIndex.isAvailable() == false) { logger.warn("failed to get access token [{}] because index [{}] is not available", userTokenId, tokensIndex.aliasName()); listener.onResponse(null); @@ -727,16 +727,23 @@ public void refreshToken(String refreshToken, ActionListener backoff, ActionListener listener) { - Tuple versionAndRefreshTokenTuple; + final Version refreshTokenVersion; + final String unencodedRefreshToken; try { - versionAndRefreshTokenTuple = unpackVersionAndPayload(refreshToken); + final Tuple versionAndRefreshTokenTuple = unpackVersionAndPayload(refreshToken); + refreshTokenVersion = versionAndRefreshTokenTuple.v1(); + unencodedRefreshToken = versionAndRefreshTokenTuple.v2(); } catch (IOException e) { // could be an old format refresh token logger.debug("refresh token could be one of the old unversioned format"); - versionAndRefreshTokenTuple = new Tuple(Version.V_7_0_0, refreshToken); + findTokenFromRefreshToken(refreshToken, securityMainIndex, backoff, listener); + return; + } + if (refreshTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { + findTokenFromRefreshToken(unencodedRefreshToken, securityTokensIndex, backoff, listener); + } else { + findTokenFromRefreshToken(refreshToken, securityMainIndex, backoff, listener); } - final SecurityIndexManager tokensIndexManager = getSecurityIndexManagerForVersion(versionAndRefreshTokenTuple.v1()); - findTokenFromRefreshToken(versionAndRefreshTokenTuple.v2(), tokensIndexManager, backoff, listener); } /** @@ -889,7 +896,7 @@ public void onFailure(Exception e) { } assert seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; assert primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM : "expected an assigned primary term"; - final SecurityIndexManager refreshedTokenIndex = getSecurityIndexManagerForVersion(refreshTokenStatus.getVersion()); + final SecurityIndexManager refreshedTokenIndex = getTokensIndexManagerForVersion(refreshTokenStatus.getVersion()); final UpdateRequestBuilder updateRequest = client .prepareUpdate(refreshedTokenIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) .setDoc("refresh_token", updateMap) @@ -979,7 +986,7 @@ private void getSupersedingTokenDocAsync(RefreshTokenStatus refreshTokenStatus, logger.debug("superseding token-doc-id in old unversioned format"); versionAndSupersedingTokenDocId = new Tuple(Version.V_7_0_0, refreshTokenStatus.getSupersedingDocId()); } - final SecurityIndexManager securityIndexManager = getSecurityIndexManagerForVersion(versionAndSupersedingTokenDocId.v1()); + final SecurityIndexManager securityIndexManager = getTokensIndexManagerForVersion(versionAndSupersedingTokenDocId.v1()); getTokenDocAsync(versionAndSupersedingTokenDocId.v2(), securityIndexManager, listener); } @@ -1337,7 +1344,7 @@ private void ensureEnabled() { * ExpiredTokenRemover} periodic job. Therefore, in general, when searching for a token we need to consider both the new and the old * indices. */ - private SecurityIndexManager getSecurityIndexManagerForVersion(Version version) { + private SecurityIndexManager getTokensIndexManagerForVersion(Version version) { if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { return securityTokensIndex; } else { @@ -1353,7 +1360,7 @@ private void checkIfTokenIsValid(UserToken userToken, ActionListener listener.onFailure(traceLog("validate token", userToken.getId(), expiredTokenException())); return; } - final SecurityIndexManager tokensIndex = getSecurityIndexManagerForVersion(userToken.getVersion()); + final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(userToken.getVersion()); if (tokensIndex.indexExists() == false) { // index doesn't exist so the token is considered invalid as we cannot verify its validity logger.warn("failed to validate access token because the security index doesn't exist"); From 44b7cc17ebc81c4b5b9ef0441fb59fb1566bd772 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 3 Apr 2019 20:55:16 +0300 Subject: [PATCH 18/50] driving on the wrong side --- .../xpack/security/authc/TokenService.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 1827e0b6acee4..84cb81708e26c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -274,13 +274,7 @@ private void createOAuth2Tokens(String userTokenId, Version version, Authenticat authentication.getLookedUpBy(), version, AuthenticationType.TOKEN, authentication.getMetadata()); final UserToken userToken = new UserToken(userTokenId, version, tokenAuth, getExpirationTime(), metadata); final String plainRefreshToken = includeRefreshToken ? UUIDs.randomBase64UUID() : null; - final String versionedRefreshToken = includeRefreshToken ? prependVersionAndEncode(version, plainRefreshToken) : null; - final BytesReference tokenDocument; - if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { - tokenDocument = createTokenDocument(userToken, plainRefreshToken, originatingClientAuth); - } else { - tokenDocument = createTokenDocument(userToken, versionedRefreshToken, originatingClientAuth); - } + final BytesReference tokenDocument = createTokenDocument(userToken, plainRefreshToken, originatingClientAuth); final String documentId = getTokenDocumentId(userToken); final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(version); final IndexRequest indexTokenRequest = client.prepareIndex(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) @@ -292,7 +286,14 @@ private void createOAuth2Tokens(String userTokenId, Version version, Authenticat () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, indexTokenRequest, ActionListener.wrap(indexResponse -> { if (indexResponse.getResult() == Result.CREATED) { - listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); + if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { + final String versionedRefreshToken = plainRefreshToken != null + ? prependVersionAndEncode(version, plainRefreshToken) + : null; + listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); + } else { + listener.onResponse(new Tuple<>(userToken, plainRefreshToken)); + } } else { listener.onFailure(traceLog("create token", new ElasticsearchException("failed to create token document [{}]", indexResponse))); From 9cd6579f10b11bb6f38f5410d0a93e4698e25e4d Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 4 Apr 2019 13:54:41 +0300 Subject: [PATCH 19/50] Remove versionCompatibility --- .../elasticsearch/xpack/security/authc/TokenService.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 84cb81708e26c..bc0ec0888b561 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -200,8 +200,6 @@ public final class TokenService { private volatile TokenKeys keyCache; private volatile long lastExpirationRunMs; private final AtomicLong createdTimeStamps = new AtomicLong(-1); - // for testing only - private final AtomicReference versionCompatibilityOverride = new AtomicReference<>(); /** * Creates a new token service @@ -231,10 +229,6 @@ public TokenService(Settings settings, Clock clock, Client client, SecurityIndex } private Version getVersionCompatibility() { - final Version versionOverride = versionCompatibilityOverride.get(); - if (versionOverride != null) { - return versionOverride; - } return clusterService.state().nodes().getMinNodeVersion(); } From 7117bc879826b316f02651beab7a7b3c1b3f0a24 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 4 Apr 2019 15:44:59 +0300 Subject: [PATCH 20/50] parseTokensFromDocument --- .../xpack/security/authc/TokenService.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index bc0ec0888b561..7d9ceb25318a9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -286,6 +286,8 @@ private void createOAuth2Tokens(String userTokenId, Version version, Authenticat : null; listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); } else { + // prior versions are not prepended, as nodes on those versions don't expect it. Such nodes + // might exist in a mixed cluster during a rolling upgrade. listener.onResponse(new Tuple<>(userToken, plainRefreshToken)); } } else { @@ -347,7 +349,7 @@ void getUserTokenFromId(String userTokenId, Version version, ActionListener listener.onFailure(traceLog("prepare security tokens index", userTokenId, ex)), + ex -> listener.onFailure(traceLog("prepare tokens index [" + tokensIndex.aliasName() +"]", userTokenId, ex)), () -> { final GetRequest getRequest = client.prepareGet(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, getTokenDocumentId(userTokenId)).request(); @@ -1305,9 +1307,14 @@ private Tuple parseTokensFromDocument(Map sou return null; } final UserToken userToken = UserToken.fromSourceMap(userTokenSource); - final String versionedRefreshToken = plainRefreshToken != null ? - prependVersionAndEncode(userToken.getVersion(), plainRefreshToken) : null; - return new Tuple<>(userToken, versionedRefreshToken); + if (userToken.getVersion().onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { + final String versionedRefreshToken = plainRefreshToken != null ? + prependVersionAndEncode(userToken.getVersion(), plainRefreshToken) : null; + return new Tuple<>(userToken, versionedRefreshToken); + } else { + // do not prepend version to refresh token as the audience node version cannot deal with it + return new Tuple<>(userToken, plainRefreshToken); + } } private static String getTokenDocumentId(UserToken userToken) { @@ -1358,7 +1365,7 @@ private void checkIfTokenIsValid(UserToken userToken, ActionListener final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(userToken.getVersion()); if (tokensIndex.indexExists() == false) { // index doesn't exist so the token is considered invalid as we cannot verify its validity - logger.warn("failed to validate access token because the security index doesn't exist"); + logger.warn("failed to validate access token because the index [" + tokensIndex.aliasName() + "] doesn't exist"); listener.onResponse(null); } else { tokensIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { From 3952d8abb380acadc481d3f5169473ad1d0a62dc Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 4 Apr 2019 16:44:42 +0300 Subject: [PATCH 21/50] tryUnpackVersionAndPayload --- .../xpack/security/authc/TokenService.java | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 7d9ceb25318a9..a88728940f51e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -724,21 +724,14 @@ public void refreshToken(String refreshToken, ActionListener backoff, ActionListener listener) { - final Version refreshTokenVersion; - final String unencodedRefreshToken; - try { - final Tuple versionAndRefreshTokenTuple = unpackVersionAndPayload(refreshToken); - refreshTokenVersion = versionAndRefreshTokenTuple.v1(); - unencodedRefreshToken = versionAndRefreshTokenTuple.v2(); - } catch (IOException e) { - // could be an old format refresh token - logger.debug("refresh token could be one of the old unversioned format"); - findTokenFromRefreshToken(refreshToken, securityMainIndex, backoff, listener); - return; - } - if (refreshTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { + final Optional> versionAndRefreshTokenTuple = tryUnpackVersionAndPayload(refreshToken); + if (versionAndRefreshTokenTuple.isPresent()) { + final Version refreshTokenVersion = versionAndRefreshTokenTuple.get().v1(); + assert refreshTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED); + final String unencodedRefreshToken = versionAndRefreshTokenTuple.get().v2(); findTokenFromRefreshToken(unencodedRefreshToken, securityTokensIndex, backoff, listener); } else { + logger.debug("Could not decode as a versioned refresh token. Assuming unversioned but valid refresh token."); findTokenFromRefreshToken(refreshToken, securityMainIndex, backoff, listener); } } @@ -975,16 +968,17 @@ public void onFailure(Exception e) { } private void getSupersedingTokenDocAsync(RefreshTokenStatus refreshTokenStatus, ActionListener listener) { - Tuple versionAndSupersedingTokenDocId; - try { - versionAndSupersedingTokenDocId = unpackVersionAndPayload(refreshTokenStatus.getSupersedingDocId()); - } catch (IOException e) { - // could be an old format refresh token - logger.debug("superseding token-doc-id in old unversioned format"); - versionAndSupersedingTokenDocId = new Tuple(Version.V_7_0_0, refreshTokenStatus.getSupersedingDocId()); + final Optional> versionAndSupersedingTokenDocId = tryUnpackVersionAndPayload( + refreshTokenStatus.getSupersedingDocId()); + if (versionAndSupersedingTokenDocId.isPresent()) { + final Version supersedingTokenVersion = versionAndSupersedingTokenDocId.get().v1(); + assert supersedingTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED); + final String supersedingTokenDocId = versionAndSupersedingTokenDocId.get().v2(); + getTokenDocAsync(supersedingTokenDocId, securityTokensIndex, listener); + } else { + logger.debug("superseding token-doc-id is not versioned. Assuming the unversioned format."); + getTokenDocAsync(refreshTokenStatus.getSupersedingDocId(), securityMainIndex, listener); } - final SecurityIndexManager securityIndexManager = getTokensIndexManagerForVersion(versionAndSupersedingTokenDocId.v1()); - getTokenDocAsync(versionAndSupersedingTokenDocId.v2(), securityIndexManager, listener); } private void getTokenDocAsync(String tokenDocId, SecurityIndexManager tokensIndex, ActionListener listener) { @@ -1312,7 +1306,7 @@ private Tuple parseTokensFromDocument(Map sou prependVersionAndEncode(userToken.getVersion(), plainRefreshToken) : null; return new Tuple<>(userToken, versionedRefreshToken); } else { - // do not prepend version to refresh token as the audience node version cannot deal with it + // do not prepend version to refresh token as the audience node version cannot deal with itqq return new Tuple<>(userToken, plainRefreshToken); } } @@ -1492,13 +1486,17 @@ private static String prependVersionAndEncode(Version version, String payload) { } } - private static Tuple unpackVersionAndPayload(String encodedPack) throws IOException { + private static Optional> tryUnpackVersionAndPayload(String encodedPack) { final byte[] bytes = encodedPack.getBytes(StandardCharsets.UTF_8); try (StreamInput in = new InputStreamStreamInput(Base64.getDecoder().wrap(new ByteArrayInputStream(bytes)), bytes.length)) { final Version version = Version.readVersion(in); in.setVersion(version); final String payload = in.readString(); - return new Tuple(version, payload); + return Optional.of(new Tuple(version, payload)); + } catch (IOException | IllegalArgumentException e) { + logger.trace("Error decoding versioned String value." + + " Probably the version is prior to the expected format, but might still be compatible.", e); + return Optional.empty(); } } From aa853291283f4b48ca29f12afd01c32b13066864 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 4 Apr 2019 18:04:42 +0300 Subject: [PATCH 22/50] getSupersedingTokenDocAsyncWithRetry --- .../xpack/security/authc/TokenService.java | 105 ++++++++++-------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index a88728940f51e..92da3628552d9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -824,55 +824,7 @@ private void innerRefresh(String tokenDocId, Map source, long se if (refreshTokenStatus.isRefreshed()) { logger.debug("Token document [{}] was recently refreshed, when a new token document [{}] was generated. Reusing that result.", tokenDocId, refreshTokenStatus.getSupersedingDocId()); - getSupersedingTokenDocAsync(refreshTokenStatus, new ActionListener() { - private final Consumer maybeRetryOnFailure = ex -> { - if (backoff.hasNext()) { - final TimeValue backofTimeValue = backoff.next(); - logger.debug("retrying after [" + backofTimeValue + "] back off"); - final Runnable retryWithContextRunnable = client.threadPool().getThreadContext() - .preserveContext(() -> getSupersedingTokenDocAsync(refreshTokenStatus, this)); - client.threadPool().schedule(retryWithContextRunnable, backofTimeValue, GENERIC); - } else { - logger.warn("back off retries exhausted"); - onFailure.accept(ex); - } - }; - - @Override - public void onResponse(GetResponse response) { - if (response.isExists()) { - logger.debug("found superseding token document [{}] for token document [{}]", - refreshTokenStatus.getSupersedingDocId(), tokenDocId); - final Tuple parsedTokens; - try { - parsedTokens = parseTokensFromDocument(response.getSource(), null); - } catch (IllegalStateException | DateTimeException e) { - logger.error("unable to decode existing user token", e); - listener.onFailure(new ElasticsearchSecurityException("could not refresh the requested token", e)); - return; - } - listener.onResponse(parsedTokens); - } else { - // We retry this since the creation of the superseding token document might already be in flight but not - // yet completed, triggered by a refresh request that came a few milliseconds ago - logger.info("could not find superseding token document [{}] for token document [{}], retrying", - refreshTokenStatus.getSupersedingDocId(), tokenDocId); - maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); - } - } - - @Override - public void onFailure(Exception e) { - if (isShardNotAvailableException(e)) { - logger.info("could not find superseding token document [{}] for refresh, retrying", - refreshTokenStatus.getSupersedingDocId()); - maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); - } else { - logger.warn("could not find superseding token document [{}] for refresh", refreshTokenStatus.getSupersedingDocId()); - onFailure.accept(invalidGrantException("could not refresh the requested token")); - } - } - }); + getSupersedingTokenDocAsyncWithRetry(refreshTokenStatus, backoff, listener); } else { final String newUserTokenId = UUIDs.randomBase64UUID(); final Version newTokenVersion = getVersionCompatibility(); @@ -967,6 +919,61 @@ public void onFailure(Exception e) { } } + private void getSupersedingTokenDocAsyncWithRetry(RefreshTokenStatus refreshTokenStatus, Iterator backoff, + ActionListener> listener) { + final Consumer onFailure = ex -> listener + .onFailure(traceLog("get superseding token", refreshTokenStatus.getSupersedingDocId(), ex)); + getSupersedingTokenDocAsync(refreshTokenStatus, new ActionListener() { + private final Consumer maybeRetryOnFailure = ex -> { + if (backoff.hasNext()) { + final TimeValue backofTimeValue = backoff.next(); + logger.debug("retrying after [" + backofTimeValue + "] back off"); + final Runnable retryWithContextRunnable = client.threadPool().getThreadContext() + .preserveContext(() -> getSupersedingTokenDocAsync(refreshTokenStatus, this)); + client.threadPool().schedule(retryWithContextRunnable, backofTimeValue, GENERIC); + } else { + logger.warn("back off retries exhausted"); + onFailure.accept(ex); + } + }; + + @Override + public void onResponse(GetResponse response) { + if (response.isExists()) { + logger.debug("found superseding token document [{}] in index [{}] by following the [{}] reference", response.getId(), + response.getIndex(), refreshTokenStatus.getSupersedingDocId()); + final Tuple parsedTokens; + try { + parsedTokens = parseTokensFromDocument(response.getSource(), null); + } catch (IllegalStateException | DateTimeException e) { + logger.error("unable to decode existing user token", e); + listener.onFailure(new ElasticsearchSecurityException("could not refresh the requested token", e)); + return; + } + listener.onResponse(parsedTokens); + } else { + // We retry this since the creation of the superseding token document might already be in flight but not + // yet completed, triggered by a refresh request that came a few milliseconds ago + logger.info("could not find superseding token document from [{}] reference, retrying", + refreshTokenStatus.getSupersedingDocId()); + maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); + } + } + + @Override + public void onFailure(Exception e) { + if (isShardNotAvailableException(e)) { + logger.info("could not find superseding token document from reference [{}], retrying", + refreshTokenStatus.getSupersedingDocId()); + maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); + } else { + logger.warn("could not find superseding token document from reference [{}]", refreshTokenStatus.getSupersedingDocId()); + onFailure.accept(invalidGrantException("could not refresh the requested token")); + } + } + }); + } + private void getSupersedingTokenDocAsync(RefreshTokenStatus refreshTokenStatus, ActionListener listener) { final Optional> versionAndSupersedingTokenDocId = tryUnpackVersionAndPayload( refreshTokenStatus.getSupersedingDocId()); From f78d22df68661ea193bfe2304e7c23935186dd2c Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 4 Apr 2019 22:33:14 +0300 Subject: [PATCH 23/50] ExpiredTokenRemover --- .../security/authc/ExpiredTokenRemover.java | 45 +++++++++++-------- .../xpack/security/authc/TokenService.java | 5 +-- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index 69ba7ce49e337..3c0543ae5f780 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.DeleteByQueryAction; @@ -38,7 +39,7 @@ * The document gets deleted if it was created more than 24 hours which is the maximum * lifetime of a refresh token */ -final class ExpiredTokenRemover { +final class ExpiredTokenRemover extends AbstractRunnable { private static final Logger logger = LogManager.getLogger(ExpiredTokenRemover.class); public static final long MAXIMUM_TOKEN_LIFETIME_HOURS = 24L; @@ -48,6 +49,7 @@ final class ExpiredTokenRemover { private final SecurityIndexManager securityTokensIndex; private final AtomicBoolean inProgress; private final TimeValue timeout; + private boolean mainIndexMightContainTokens; ExpiredTokenRemover(Settings settings, Client client, SecurityIndexManager securityMainIndex, SecurityIndexManager securityTokensIndex) { @@ -56,10 +58,22 @@ final class ExpiredTokenRemover { this.securityTokensIndex = securityTokensIndex; this.inProgress = new AtomicBoolean(false); this.timeout = TokenService.DELETE_TIMEOUT.get(settings); + this.mainIndexMightContainTokens = true; } - public void doRun(String... tokensIndexNames) { - DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(tokensIndexNames); + @Override + public void doRun() { + final List indicesWithTokens = new ArrayList<>(); + if (securityTokensIndex.isAvailable()) { + indicesWithTokens.add(securityTokensIndex.aliasName()); + } + if (securityMainIndex.isAvailable() && mainIndexMightContainTokens) { + indicesWithTokens.add(securityMainIndex.aliasName()); + } + if (indicesWithTokens.isEmpty()) { + return; + } + DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(indicesWithTokens.toArray(new String[0])); if (timeout != TimeValue.MINUS_ONE) { expiredDbq.setTimeout(timeout); expiredDbq.getSearchRequest().source().timeout(timeout); @@ -74,27 +88,21 @@ public void doRun(String... tokensIndexNames) { executeAsyncWithOrigin(client, SECURITY_ORIGIN, DeleteByQueryAction.INSTANCE, expiredDbq, ActionListener.wrap(r -> { debugDbqResponse(r); + // tokens can still linger on the main index for their maximum lifetime after the tokens index has been created, + // because when tokens index has been created all nodes will store tokens there and not on the main security index + if (mainIndexMightContainTokens && securityTokensIndex.indexExists() + && securityTokensIndex.getCreationTime().isBefore(now.minus(MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS)) + && r.getBulkFailures().isEmpty() && r.getSearchFailures().isEmpty()) { + mainIndexMightContainTokens = false; + } markComplete(); }, this::onFailure)); } - boolean submit(ThreadPool threadPool) { - final List indicesWithTokens = new ArrayList<>(); - if (securityTokensIndex.isAvailable()) { - indicesWithTokens.add(securityTokensIndex.aliasName()); - } - if (securityMainIndex.isAvailable() && (false == securityTokensIndex.indexExists() - || Instant.now().minus(MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS).isBefore(securityTokensIndex.getCreationTime()))) { - indicesWithTokens.add(securityMainIndex.aliasName()); - } - if (indicesWithTokens.isEmpty()) { - return false; - } + void submit(ThreadPool threadPool) { if (inProgress.compareAndSet(false, true)) { - threadPool.executor(Names.GENERIC).submit(() -> doRun(indicesWithTokens.toArray(new String[0]))); - return true; + threadPool.executor(Names.GENERIC).submit(() -> doRun()); } - return false; } private void debugDbqResponse(BulkByScrollResponse response) { @@ -116,6 +124,7 @@ boolean isExpirationInProgress() { return inProgress.get(); } + @Override public void onFailure(Exception e) { if (isShardNotAvailableException(e)) { logger.debug("failed to delete expired tokens", e); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 92da3628552d9..3765c0596b019 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -1422,9 +1422,8 @@ private Instant getCreationTime(Instant expire) { private void maybeStartTokenRemover() { if (client.threadPool().relativeTimeInMillis() - lastExpirationRunMs > deleteInterval.getMillis()) { - if (expiredTokenRemover.submit(client.threadPool())) { - lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); - } + expiredTokenRemover.submit(client.threadPool()); + lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); } } From 13903a79c0633bac8557950541ae10d4401f829e Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 4 Apr 2019 22:34:27 +0300 Subject: [PATCH 24/50] Nit --- .../xpack/security/authc/ExpiredTokenRemover.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index 3c0543ae5f780..bc6359cb4a86b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -88,8 +88,8 @@ public void doRun() { executeAsyncWithOrigin(client, SECURITY_ORIGIN, DeleteByQueryAction.INSTANCE, expiredDbq, ActionListener.wrap(r -> { debugDbqResponse(r); - // tokens can still linger on the main index for their maximum lifetime after the tokens index has been created, - // because when tokens index has been created all nodes will store tokens there and not on the main security index + // tokens can still linger on the main index for their maximum lifetime after the tokens index has been created, because + // only after the tokens index has been created all nodes will store tokens there and not on the main security index if (mainIndexMightContainTokens && securityTokensIndex.indexExists() && securityTokensIndex.getCreationTime().isBefore(now.minus(MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS)) && r.getBulkFailures().isEmpty() && r.getSearchFailures().isEmpty()) { From 4e135c34894544b8753ce0968f154f3c8a0e9931 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Apr 2019 00:54:55 +0300 Subject: [PATCH 25/50] sourceTokenIndicesStateAndRun --- .../xpack/security/authc/TokenService.java | 179 ++++++++---------- .../support/SecurityIndexManager.java | 4 + 2 files changed, 85 insertions(+), 98 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 3765c0596b019..40197cd1b2384 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -1108,63 +1108,39 @@ public void findActiveTokensForRealm(String realmName, @Nullable Predicate>> listener) { ensureEnabled(); if (Strings.isNullOrEmpty(realmName)) { - listener.onFailure(new IllegalArgumentException("Realm name is required")); + listener.onFailure(new IllegalArgumentException("realm name is required")); return; } - final Instant now = clock.instant(); - findActiveTokensForRealm(realmName, securityTokensIndex, now, filter, ActionListener.wrap(tokens -> { - if (false == securityTokensIndex.indexExists() - || Instant.now().minus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS) - .isBefore(securityTokensIndex.getCreationTime())) { - findActiveTokensForRealm(realmName, securityMainIndex, now, filter, ActionListener.wrap(tokensInSecurityMain -> { - if (false == tokensInSecurityMain.isEmpty()) { - final List> allTokens = new ArrayList<>(); - allTokens.addAll(tokens); - allTokens.addAll(tokensInSecurityMain); - listener.onResponse(Collections.unmodifiableList(allTokens)); - } else { - listener.onResponse(tokens); - } - }, listener::onFailure)); + sourceTokenIndicesStateAndRun(ActionListener.wrap(indicesWithTokens -> { + if (indicesWithTokens.isEmpty()) { + listener.onResponse(Collections.emptyList()); } else { - listener.onResponse(tokens); + final Instant now = clock.instant(); + final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) + .filter(QueryBuilders.termQuery("access_token.realm", realmName)) + .filter(QueryBuilders.boolQuery() + .should(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("access_token.invalidated", false)) + .must(QueryBuilders.rangeQuery("access_token.user_token.expiration_time").gte(now.toEpochMilli())) + ) + .should(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("refresh_token.invalidated", false)) + .must(QueryBuilders.rangeQuery("creation_time").gte(now.toEpochMilli() + - TimeValue.timeValueHours(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS).millis())) + ) + ); + final SearchRequest request = client.prepareSearch(indicesWithTokens.toArray(new String[0])) + .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) + .setQuery(boolQuery) + .setVersion(false) + .setSize(1000) + .setFetchSource(true) + .request(); + ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, filter)); } }, listener::onFailure)); - } - private void findActiveTokensForRealm(String realmName, SecurityIndexManager tokensIndex, Instant now, - @Nullable Predicate> filter, - ActionListener>> listener) { - final SecurityIndexManager frozenTokensIndex = tokensIndex.freeze(); - if (frozenTokensIndex.indexExists() == false) { - listener.onResponse(Collections.emptyList()); - } else if (frozenTokensIndex.isAvailable() == false) { - listener.onFailure(frozenTokensIndex.getUnavailableReason()); - } else { - final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) - .filter(QueryBuilders.termQuery("access_token.realm", realmName)) - .filter(QueryBuilders.boolQuery() - .should(QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("access_token.invalidated", false)) - .must(QueryBuilders.rangeQuery("access_token.user_token.expiration_time").gte(now.toEpochMilli())) - ) - .should(QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("refresh_token.invalidated", false)) - .must(QueryBuilders.rangeQuery("creation_time").gte(now.toEpochMilli() - - TimeValue.timeValueHours(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS).millis())) - ) - ); - final SearchRequest request = client.prepareSearch(tokensIndex.aliasName()) - .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) - .setQuery(boolQuery) - .setVersion(false) - .setSize(1000) - .setFetchSource(true) - .request(); - tokensIndex.checkIndexVersionThenExecute(listener::onFailure, - () -> ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, filter))); - } } /** @@ -1180,59 +1156,66 @@ public void findActiveTokensForUser(String username, ActionListener { - if (false == securityTokensIndex.indexExists() - || Instant.now().minus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS) - .isBefore(securityTokensIndex.getCreationTime())) { - findActiveTokensForUser(username, securityMainIndex, now, ActionListener.wrap(tokensInSecurityMain -> { - if (false == tokensInSecurityMain.isEmpty()) { - final List> allTokens = new ArrayList<>(); - allTokens.addAll(tokens); - allTokens.addAll(tokensInSecurityMain); - listener.onResponse(Collections.unmodifiableList(allTokens)); - } else { - listener.onResponse(tokens); - } - }, listener::onFailure)); + sourceTokenIndicesStateAndRun(ActionListener.wrap(indicesWithTokens -> { + if (indicesWithTokens.isEmpty()) { + listener.onResponse(Collections.emptyList()); } else { - listener.onResponse(tokens); + final Instant now = clock.instant(); + final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) + .filter(QueryBuilders.boolQuery() + .should(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("access_token.invalidated", false)) + .must(QueryBuilders.rangeQuery("access_token.user_token.expiration_time").gte(now.toEpochMilli())) + ) + .should(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("refresh_token.invalidated", false)) + .must(QueryBuilders.rangeQuery("creation_time").gte(now.toEpochMilli() + - TimeValue.timeValueHours(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS).millis())) + ) + ); + final SearchRequest request = client.prepareSearch(indicesWithTokens.toArray(new String[0])) + .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) + .setQuery(boolQuery) + .setVersion(false) + .setSize(1000) + .setFetchSource(true) + .request(); + ScrollHelper.fetchAllByEntity(client, request, listener, (SearchHit hit) -> filterAndParseHit(hit, isOfUser(username))); } }, listener::onFailure)); } - private void findActiveTokensForUser(String username, SecurityIndexManager tokensIndex, Instant now, - ActionListener>> listener) { - final SecurityIndexManager frozenTokensIndex = tokensIndex.freeze(); - if (frozenTokensIndex.indexExists() == false) { - listener.onResponse(Collections.emptyList()); - } else if (frozenTokensIndex.isAvailable() == false) { - listener.onFailure(frozenTokensIndex.getUnavailableReason()); - } else { - final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery("doc_type", TOKEN_DOC_TYPE)) - .filter(QueryBuilders.boolQuery() - .should(QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("access_token.invalidated", false)) - .must(QueryBuilders.rangeQuery("access_token.user_token.expiration_time").gte(now.toEpochMilli())) - ) - .should(QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("refresh_token.invalidated", false)) - .must(QueryBuilders.rangeQuery("creation_time").gte(now.toEpochMilli() - - TimeValue.timeValueHours(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS).millis())) - ) - ); - final SearchRequest request = client.prepareSearch(tokensIndex.aliasName()) - .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings)) - .setQuery(boolQuery) - .setVersion(false) - .setSize(1000) - .setFetchSource(true) - .request(); - tokensIndex.checkIndexVersionThenExecute(listener::onFailure, - () -> ScrollHelper.fetchAllByEntity(client, request, listener, - (SearchHit hit) -> filterAndParseHit(hit, isOfUser(username)))); + private void sourceTokenIndicesStateAndRun(ActionListener> listener) { + final List indicesWithTokens = new ArrayList<>(2); + final SecurityIndexManager frozenTokensIndex = securityTokensIndex.freeze(); + if (frozenTokensIndex.indexExists()) { + if (false == frozenTokensIndex.isAvailable()) { + listener.onFailure(frozenTokensIndex.getUnavailableReason()); + return; + } else if (false == frozenTokensIndex.isIndexUpToDate()) { + listener.onFailure(new IllegalStateException( + "Index [" + frozenTokensIndex.aliasName() + "] is not on the current version. Features relying on the index" + + " will not be available until the upgrade API is run on the index")); + } else { + indicesWithTokens.add(frozenTokensIndex.aliasName()); + } + } + final SecurityIndexManager frozenMainIndex = securityMainIndex.freeze(); + if (frozenMainIndex.indexExists()) { + if (false == frozenMainIndex.isAvailable()) { + listener.onFailure(frozenMainIndex.getUnavailableReason()); + return; + } else if (false == frozenMainIndex.isIndexUpToDate()) { + listener.onFailure(new IllegalStateException( + "Index [" + frozenMainIndex.aliasName() + "] is not on the current version. Features relying on the index" + + " will not be available until the upgrade API is run on the index")); + } else if (false == frozenTokensIndex.indexExists() || frozenTokensIndex.getCreationTime() + .isBefore(clock.instant().minus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS))) { + indicesWithTokens.add(frozenMainIndex.aliasName()); + } } + listener.onResponse(indicesWithTokens); } private BytesReference createTokenDocument(UserToken userToken, @Nullable String refreshToken, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 12a890dcbe6cd..04fbf974c6f0a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -159,6 +159,10 @@ public boolean isAvailable() { return this.indexState.indexAvailable; } + public boolean indexExistsButUnavailable() { + return indexExists() && false == isAvailable(); + } + public boolean isMappingUpToDate() { return this.indexState.mappingUpToDate; } From c4f04eabf222924c58516c25a5a2026c23324372 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Apr 2019 00:55:26 +0300 Subject: [PATCH 26/50] Checkstyle --- .../org/elasticsearch/xpack/security/authc/TokenService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 40197cd1b2384..71c0c86316b45 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -130,7 +130,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; From 8630a7ff5aaf24287bb53146b5215338d9ec09f4 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Apr 2019 01:01:15 +0300 Subject: [PATCH 27/50] prepareIndexIfNeededThenExecute with aliad name --- .../org/elasticsearch/xpack/security/authc/TokenService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 71c0c86316b45..fb57b7d855711 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -275,7 +275,8 @@ private void createOAuth2Tokens(String userTokenId, Version version, Authenticat .setSource(tokenDocument, XContentType.JSON) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .request(); - tokensIndex.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security tokens index", documentId, ex)), + tokensIndex.prepareIndexIfNeededThenExecute( + ex -> listener.onFailure(traceLog("prepare tokens index [" + tokensIndex.aliasName() + "]", documentId, ex)), () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, indexTokenRequest, ActionListener.wrap(indexResponse -> { if (indexResponse.getResult() == Result.CREATED) { From 9052b52eaaf141424c3ab277abbdec5cdc36a142 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Apr 2019 13:29:39 +0300 Subject: [PATCH 28/50] threadPool.executor(Names.GENERIC).submit(this); --- .../elasticsearch/xpack/security/authc/ExpiredTokenRemover.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index bc6359cb4a86b..5d57106df1d43 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -101,7 +101,7 @@ public void doRun() { void submit(ThreadPool threadPool) { if (inProgress.compareAndSet(false, true)) { - threadPool.executor(Names.GENERIC).submit(() -> doRun()); + threadPool.executor(Names.GENERIC).submit(this); } } From 7b4e687dfcf4fba2734bf240699f48b069e1f42f Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Apr 2019 15:17:37 +0300 Subject: [PATCH 29/50] Address review --- .../xpack/security/authc/TokenService.java | 96 +++++++++++-------- 1 file changed, 57 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index fb57b7d855711..e55b0bafb1f71 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -70,6 +70,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.gateway.AsyncShardFetch.Lister; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -183,6 +184,8 @@ public final class TokenService { static final Version VERSION_TOKENS_INDEX_INTRODUCED = Version.V_8_0_0; // TODO change upon backport static final Version VERSION_ACCESS_TOKENS_AS_UUIDS = Version.V_7_1_0; static final Version VERSION_MULTIPLE_CONCURRENT_REFRESHES = Version.V_7_1_0; + // UUIDs are 16 bytes encoded base64 without padding, therefore the length is (16 / 3) * 4 + ((16 % 3) * 8 + 5) / 6 chars + private static final int TOKEN_ID_LENGTH = 22; private static final Logger logger = LogManager.getLogger(TokenService.class); private final SecureRandom secureRandom = new SecureRandom(); @@ -244,18 +247,23 @@ public void createOAuth2Tokens(Authentication authentication, Authentication ori ActionListener> listener) { // the created token is compatible with the oldest node version in the cluster final Version tokenVersion = getVersionCompatibility(); + // in newer versions tokens moved to a separate index + final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(tokenVersion); // the id of the created tokens ought be unguessable final String userTokenId = UUIDs.randomBase64UUID(); - createOAuth2Tokens(userTokenId, tokenVersion, authentication, originatingClientAuth, metadata, includeRefreshToken, listener); + createOAuth2Tokens(userTokenId, tokenVersion, tokensIndex, authentication, originatingClientAuth, metadata, includeRefreshToken, + listener); } /** * Create an access token and optionally a refresh token as well, based on the provided authentication and metadata, with the given * token document id. The created tokens are be stored in the security index. */ - private void createOAuth2Tokens(String userTokenId, Version version, Authentication authentication, - Authentication originatingClientAuth, Map metadata, - boolean includeRefreshToken, ActionListener> listener) { + private void createOAuth2Tokens(String userTokenId, Version version, SecurityIndexManager tokensIndex, Authentication authentication, + Authentication originatingClientAuth, Map metadata, boolean includeRefreshToken, + ActionListener> listener) { + assert userTokenId.length() == TOKEN_ID_LENGTH : "We assume token ids have a fixed length for nodes of a certain version." + + " When changing this be careful that the inferences from token length still hold."; ensureEnabled(); if (authentication == null) { listener.onFailure(traceLog("create token", new IllegalArgumentException("authentication must be provided"))); @@ -269,7 +277,6 @@ private void createOAuth2Tokens(String userTokenId, Version version, Authenticat final String plainRefreshToken = includeRefreshToken ? UUIDs.randomBase64UUID() : null; final BytesReference tokenDocument = createTokenDocument(userToken, plainRefreshToken, originatingClientAuth); final String documentId = getTokenDocumentId(userToken); - final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(version); final IndexRequest indexTokenRequest = client.prepareIndex(tokensIndex.aliasName(), SINGLE_MAPPING_NAME, documentId) .setOpType(OpType.CREATE) .setSource(tokenDocument, XContentType.JSON) @@ -724,15 +731,25 @@ public void refreshToken(String refreshToken, ActionListener backoff, ActionListener listener) { - final Optional> versionAndRefreshTokenTuple = tryUnpackVersionAndPayload(refreshToken); - if (versionAndRefreshTokenTuple.isPresent()) { - final Version refreshTokenVersion = versionAndRefreshTokenTuple.get().v1(); - assert refreshTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED); - final String unencodedRefreshToken = versionAndRefreshTokenTuple.get().v2(); - findTokenFromRefreshToken(unencodedRefreshToken, securityTokensIndex, backoff, listener); - } else { - logger.debug("Could not decode as a versioned refresh token. Assuming unversioned but valid refresh token."); + if (refreshToken.length() == TOKEN_ID_LENGTH) { + logger.debug("Assuming an unversioned refresh token [{}}, generated for node versions" + + " prior to the introduction of the version-header format.", refreshToken); findTokenFromRefreshToken(refreshToken, securityMainIndex, backoff, listener); + } else { + final Optional> versionAndRefreshTokenTuple = tryUnpackVersionAndPayload(refreshToken); + if (versionAndRefreshTokenTuple.isPresent()) { + final Version refreshTokenVersion = versionAndRefreshTokenTuple.get().v1(); + final String unencodedRefreshToken = versionAndRefreshTokenTuple.get().v2(); + if (false == refreshTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED) + || unencodedRefreshToken.length() != TOKEN_ID_LENGTH) { + listener.onFailure(new IllegalArgumentException("Decoded refresh token [" + unencodedRefreshToken + "] with version [" + + refreshTokenVersion + "] is invalid.")); + } else { + findTokenFromRefreshToken(unencodedRefreshToken, securityTokensIndex, backoff, listener); + } + } else { + listener.onFailure(new IllegalArgumentException("Could not decode refresh token [" + refreshToken + "].")); + } } } @@ -823,7 +840,7 @@ private void innerRefresh(String tokenDocId, Map source, long se final RefreshTokenStatus refreshTokenStatus = checkRefreshResult.v1(); if (refreshTokenStatus.isRefreshed()) { logger.debug("Token document [{}] was recently refreshed, when a new token document [{}] was generated. Reusing that result.", - tokenDocId, refreshTokenStatus.getSupersedingDocId()); + tokenDocId, refreshTokenStatus.getSupersededBy()); getSupersedingTokenDocAsyncWithRetry(refreshTokenStatus, backoff, listener); } else { final String newUserTokenId = UUIDs.randomBase64UUID(); @@ -832,8 +849,12 @@ private void innerRefresh(String tokenDocId, Map source, long se updateMap.put("refreshed", true); updateMap.put("refresh_time", clock.instant().toEpochMilli()); if (newTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { - updateMap.put("superseded_by", prependVersionAndEncode(newTokenVersion, getTokenDocumentId(newUserTokenId))); + // the superseding token document reference is formated as: "|" ; the alias points to a single index + // containing the document with the said id + updateMap.put("superseded_by", + getTokensIndexManagerForVersion(newTokenVersion).aliasName() + "|" + getTokenDocumentId(newUserTokenId)); } else { + // preservers the format of the reference so that old nodes in a mixed cluster can still understand it updateMap.put("superseded_by", getTokenDocumentId(newUserTokenId)); } assert seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; @@ -853,8 +874,8 @@ private void innerRefresh(String tokenDocId, Map source, long se updateResponse.getGetResult().sourceAsMap())); final Tuple parsedTokens = parseTokensFromDocument(source, null); final UserToken toRefreshUserToken = parsedTokens.v1(); - createOAuth2Tokens(newUserTokenId, newTokenVersion, toRefreshUserToken.getAuthentication(), clientAuth, - toRefreshUserToken.getMetadata(), true, listener); + createOAuth2Tokens(newUserTokenId, newTokenVersion, getTokensIndexManagerForVersion(newTokenVersion), + toRefreshUserToken.getAuthentication(), clientAuth, toRefreshUserToken.getMetadata(), true, listener); } else if (backoff.hasNext()) { logger.info("failed to update the original token document [{}], the update result was [{}]. Retrying", tokenDocId, updateResponse.getResult()); @@ -922,7 +943,7 @@ public void onFailure(Exception e) { private void getSupersedingTokenDocAsyncWithRetry(RefreshTokenStatus refreshTokenStatus, Iterator backoff, ActionListener> listener) { final Consumer onFailure = ex -> listener - .onFailure(traceLog("get superseding token", refreshTokenStatus.getSupersedingDocId(), ex)); + .onFailure(traceLog("get superseding token", refreshTokenStatus.getSupersededBy(), ex)); getSupersedingTokenDocAsync(refreshTokenStatus, new ActionListener() { private final Consumer maybeRetryOnFailure = ex -> { if (backoff.hasNext()) { @@ -941,7 +962,7 @@ private void getSupersedingTokenDocAsyncWithRetry(RefreshTokenStatus refreshToke public void onResponse(GetResponse response) { if (response.isExists()) { logger.debug("found superseding token document [{}] in index [{}] by following the [{}] reference", response.getId(), - response.getIndex(), refreshTokenStatus.getSupersedingDocId()); + response.getIndex(), refreshTokenStatus.getSupersededBy()); final Tuple parsedTokens; try { parsedTokens = parseTokensFromDocument(response.getSource(), null); @@ -955,7 +976,7 @@ public void onResponse(GetResponse response) { // We retry this since the creation of the superseding token document might already be in flight but not // yet completed, triggered by a refresh request that came a few milliseconds ago logger.info("could not find superseding token document from [{}] reference, retrying", - refreshTokenStatus.getSupersedingDocId()); + refreshTokenStatus.getSupersededBy()); maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); } } @@ -964,10 +985,10 @@ public void onResponse(GetResponse response) { public void onFailure(Exception e) { if (isShardNotAvailableException(e)) { logger.info("could not find superseding token document from reference [{}], retrying", - refreshTokenStatus.getSupersedingDocId()); + refreshTokenStatus.getSupersededBy()); maybeRetryOnFailure.accept(invalidGrantException("could not refresh the requested token")); } else { - logger.warn("could not find superseding token document from reference [{}]", refreshTokenStatus.getSupersedingDocId()); + logger.warn("could not find superseding token document from reference [{}]", refreshTokenStatus.getSupersededBy()); onFailure.accept(invalidGrantException("could not refresh the requested token")); } } @@ -975,16 +996,14 @@ public void onFailure(Exception e) { } private void getSupersedingTokenDocAsync(RefreshTokenStatus refreshTokenStatus, ActionListener listener) { - final Optional> versionAndSupersedingTokenDocId = tryUnpackVersionAndPayload( - refreshTokenStatus.getSupersedingDocId()); - if (versionAndSupersedingTokenDocId.isPresent()) { - final Version supersedingTokenVersion = versionAndSupersedingTokenDocId.get().v1(); - assert supersedingTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED); - final String supersedingTokenDocId = versionAndSupersedingTokenDocId.get().v2(); - getTokenDocAsync(supersedingTokenDocId, securityTokensIndex, listener); + final String supersedingDocReference = refreshTokenStatus.getSupersededBy(); + if (supersedingDocReference.startsWith(securityTokensIndex.aliasName() + "|")) { + final String supersedingDocId = supersedingDocReference.substring(securityTokensIndex.aliasName().length() + 1); + getTokenDocAsync(supersedingDocId, securityTokensIndex, listener); } else { - logger.debug("superseding token-doc-id is not versioned. Assuming the unversioned format."); - getTokenDocAsync(refreshTokenStatus.getSupersedingDocId(), securityMainIndex, listener); + assert false == supersedingDocReference + .contains("|") : "Superseding doc reference appears to contain an alias name but shouldn't"; + getTokenDocAsync(supersedingDocReference, securityMainIndex, listener); } } @@ -1296,7 +1315,7 @@ private Tuple parseTokensFromDocument(Map sou prependVersionAndEncode(userToken.getVersion(), plainRefreshToken) : null; return new Tuple<>(userToken, versionedRefreshToken); } else { - // do not prepend version to refresh token as the audience node version cannot deal with itqq + // do not prepend version to refresh token as the audience node version cannot deal with it return new Tuple<>(userToken, plainRefreshToken); } } @@ -1483,8 +1502,7 @@ private static Optional> tryUnpackVersionAndPayload(Strin final String payload = in.readString(); return Optional.of(new Tuple(version, payload)); } catch (IOException | IllegalArgumentException e) { - logger.trace("Error decoding versioned String value." - + " Probably the version is prior to the expected format, but might still be compatible.", e); + logger.trace("Error decoding versioned String value.", e); return Optional.empty(); } } @@ -2026,17 +2044,17 @@ private static final class RefreshTokenStatus { private final String associatedRealm; private final boolean refreshed; @Nullable private final Instant refreshInstant; - @Nullable private final String supersededByDocId; + @Nullable private final String supersededBy; private Version version; private RefreshTokenStatus(boolean invalidated, String associatedUser, String associatedRealm, boolean refreshed, - Instant refreshInstant, String supersededByDocId) { + Instant refreshInstant, String supersededBy) { this.invalidated = invalidated; this.associatedUser = associatedUser; this.associatedRealm = associatedRealm; this.refreshed = refreshed; this.refreshInstant = refreshInstant; - this.supersededByDocId = supersededByDocId; + this.supersededBy = supersededBy; } boolean isInvalidated() { @@ -2059,8 +2077,8 @@ boolean isRefreshed() { return refreshInstant; } - @Nullable String getSupersedingDocId() { - return supersededByDocId; + @Nullable String getSupersededBy() { + return supersededBy; } Version getVersion() { From f78927a334dc3e65f664ded8db561b9c612f6a14 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Apr 2019 15:26:59 +0300 Subject: [PATCH 30/50] Javadoc --- .../org/elasticsearch/xpack/security/authc/TokenService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index e55b0bafb1f71..326243ff21407 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -727,8 +727,9 @@ public void refreshToken(String refreshToken, ActionListener backoff, ActionListener listener) { if (refreshToken.length() == TOKEN_ID_LENGTH) { From c39162083248cadd6537d036685448a9ad50a9d2 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Apr 2019 15:35:42 +0300 Subject: [PATCH 31/50] Nits for ExpiredTokenRemover --- .../xpack/security/authc/ExpiredTokenRemover.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index 5d57106df1d43..8f06973f3423e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -35,9 +35,8 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; /** - * Responsible for cleaning the invalidated and expired tokens from the security index. - * The document gets deleted if it was created more than 24 hours which is the maximum - * lifetime of a refresh token + * Responsible for cleaning the invalidated and expired tokens from the security indices (`main` and `tokens`). + * The document is deleted if it was created more than {@code #MAXIMUM_TOKEN_LIFETIME_HOURS} hours in the past. */ final class ExpiredTokenRemover extends AbstractRunnable { private static final Logger logger = LogManager.getLogger(ExpiredTokenRemover.class); @@ -86,13 +85,13 @@ public void doRun() { .lte(now.minus(MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS).toEpochMilli()))); logger.trace(() -> new ParameterizedMessage("Removing old tokens: [{}]", Strings.toString(expiredDbq))); executeAsyncWithOrigin(client, SECURITY_ORIGIN, DeleteByQueryAction.INSTANCE, expiredDbq, - ActionListener.wrap(r -> { - debugDbqResponse(r); + ActionListener.wrap(bulkResponse -> { + debugDbqResponse(bulkResponse); // tokens can still linger on the main index for their maximum lifetime after the tokens index has been created, because // only after the tokens index has been created all nodes will store tokens there and not on the main security index if (mainIndexMightContainTokens && securityTokensIndex.indexExists() && securityTokensIndex.getCreationTime().isBefore(now.minus(MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS)) - && r.getBulkFailures().isEmpty() && r.getSearchFailures().isEmpty()) { + && bulkResponse.getBulkFailures().isEmpty() && bulkResponse.getSearchFailures().isEmpty()) { mainIndexMightContainTokens = false; } markComplete(); From 71544cb2c566a70c14845ab8bf9b3375e14ffd60 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 7 Apr 2019 12:53:10 +0300 Subject: [PATCH 32/50] Nits --- .../xpack/security/support/SecurityIndexManager.java | 3 ++- .../xpack/security/authz/AuthorizationServiceTests.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 04fbf974c6f0a..6fe57eda4c576 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -74,7 +74,8 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; /** - * Manages the lifecycle, mapping and data upgrades/migrations of one of `.security-7` or `.security-tokens-7` indices. + * Manages the lifecycle, mapping and data upgrades/migrations of {@code RestrictedIndicesNames#SECURITY_MAIN_ALIAS} and + * {@code RestrictedIndicesNames#SECURITY_MAIN_ALIAS} alias-index pairs. */ public class SecurityIndexManager implements ClusterStateListener { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index c79a7d079567a..b370c8e2b6b0f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -1039,7 +1039,7 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() .put(new IndexMetaData.Builder(INTERNAL_SECURITY_MAIN_INDEX_7) - .putAlias(new AliasMetaData.Builder(INTERNAL_SECURITY_MAIN_INDEX_7).build()) + .putAlias(new AliasMetaData.Builder(SECURITY_MAIN_ALIAS).build()) .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) .numberOfShards(1) .numberOfReplicas(0) @@ -1051,7 +1051,7 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() SearchRequest request = new SearchRequest("_all"); authorize(authentication, action, request); verify(auditTrail).accessGranted(eq(requestId), eq(authentication), eq(action), eq(request), authzInfoRoles(superuser.roles())); - assertThat(request.indices(), arrayContainingInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_SECURITY_MAIN_INDEX_7)); + assertThat(request.indices(), arrayContainingInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, SECURITY_MAIN_ALIAS)); } public void testCompositeActionsAreImmediatelyRejected() { From fb938a3094464e3e0dcc2cafcab25e3d3a30f5ce Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 7 Apr 2019 14:59:16 +0300 Subject: [PATCH 33/50] sourceIndicesWithTokensAndRun --- .../xpack/security/authc/TokenService.java | 67 ++++++++++++------- .../support/SecurityIndexManager.java | 4 -- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 326243ff21407..ebb685be697a7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -70,7 +70,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.gateway.AsyncShardFetch.Lister; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -230,14 +229,6 @@ public TokenService(Settings settings, Clock clock, Client client, SecurityIndex getTokenMetaData(); } - private Version getVersionCompatibility() { - return clusterService.state().nodes().getMinNodeVersion(); - } - - public static Boolean isTokenServiceEnabled(Settings settings) { - return XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get(settings); - } - /** * Creates an access token and optionally a refresh token as well, based on the provided authentication and metadata with an * auto-generated token document id. The created tokens are stored in the security index. @@ -246,7 +237,7 @@ public void createOAuth2Tokens(Authentication authentication, Authentication ori Map metadata, boolean includeRefreshToken, ActionListener> listener) { // the created token is compatible with the oldest node version in the cluster - final Version tokenVersion = getVersionCompatibility(); + final Version tokenVersion = getTokenVersionCompatibility(); // in newer versions tokens moved to a separate index final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(tokenVersion); // the id of the created tokens ought be unguessable @@ -845,7 +836,7 @@ private void innerRefresh(String tokenDocId, Map source, long se getSupersedingTokenDocAsyncWithRetry(refreshTokenStatus, backoff, listener); } else { final String newUserTokenId = UUIDs.randomBase64UUID(); - final Version newTokenVersion = getVersionCompatibility(); + final Version newTokenVersion = getTokenVersionCompatibility(); final Map updateMap = new HashMap<>(); updateMap.put("refreshed", true); updateMap.put("refresh_time", clock.instant().toEpochMilli()); @@ -1013,6 +1004,14 @@ private void getTokenDocAsync(String tokenDocId, SecurityIndexManager tokensInde executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, listener, client::get); } + private Version getTokenVersionCompatibility() { + return clusterService.state().nodes().getMinNodeVersion(); + } + + public static Boolean isTokenServiceEnabled(Settings settings) { + return XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get(settings); + } + /** * A refresh token has a fixed maximum lifetime of {@code ExpiredTokenRemover#MAXIMUM_TOKEN_LIFETIME_HOURS} hours. This checks if the * token document represents a valid token wrt this time interval. @@ -1131,7 +1130,7 @@ public void findActiveTokensForRealm(String realmName, @Nullable Predicate { + sourceIndicesWithTokensAndRun(ActionListener.wrap(indicesWithTokens -> { if (indicesWithTokens.isEmpty()) { listener.onResponse(Collections.emptyList()); } else { @@ -1176,7 +1175,7 @@ public void findActiveTokensForUser(String username, ActionListener { + sourceIndicesWithTokensAndRun(ActionListener.wrap(indicesWithTokens -> { if (indicesWithTokens.isEmpty()) { listener.onResponse(Collections.emptyList()); } else { @@ -1206,32 +1205,48 @@ public void findActiveTokensForUser(String username, ActionListener> listener) { + /** + * Security tokens were traditionally stored on the main security index but after version {@code #VERSION_TOKENS_INDEX_INTRODUCED} they + * have been stored on a dedicated separate index. This move has been implemented without requiring user intervention, so the newly + * created tokens started to be created in the new index, while the old tokens were still usable out of the main security index, subject + * to their maximum lifetime of {@code ExpiredTokenRemover#MAXIMUM_TOKEN_LIFETIME_HOURS} hours. Once the dedicated tokens index has been + * created, all newly created tokens will be stored inside it. This function returns the list of the indices names that might contain + * tokens. Unless there are availability or version issues, the dedicated tokens index always contains tokens. The main security index + * might contain tokens if the tokens index has not been created yet, or if it has been created recently so that there might + * still be tokens that have not yet exceeded their maximum lifetime. + */ + private void sourceIndicesWithTokensAndRun(ActionListener> listener) { final List indicesWithTokens = new ArrayList<>(2); final SecurityIndexManager frozenTokensIndex = securityTokensIndex.freeze(); if (frozenTokensIndex.indexExists()) { + // an existing tokens index always contains tokens (if available and version allows) if (false == frozenTokensIndex.isAvailable()) { listener.onFailure(frozenTokensIndex.getUnavailableReason()); return; - } else if (false == frozenTokensIndex.isIndexUpToDate()) { + } + if (false == frozenTokensIndex.isIndexUpToDate()) { listener.onFailure(new IllegalStateException( "Index [" + frozenTokensIndex.aliasName() + "] is not on the current version. Features relying on the index" + " will not be available until the upgrade API is run on the index")); - } else { - indicesWithTokens.add(frozenTokensIndex.aliasName()); + return; } + indicesWithTokens.add(frozenTokensIndex.aliasName()); } final SecurityIndexManager frozenMainIndex = securityMainIndex.freeze(); if (frozenMainIndex.indexExists()) { - if (false == frozenMainIndex.isAvailable()) { - listener.onFailure(frozenMainIndex.getUnavailableReason()); - return; - } else if (false == frozenMainIndex.isIndexUpToDate()) { - listener.onFailure(new IllegalStateException( - "Index [" + frozenMainIndex.aliasName() + "] is not on the current version. Features relying on the index" - + " will not be available until the upgrade API is run on the index")); - } else if (false == frozenTokensIndex.indexExists() || frozenTokensIndex.getCreationTime() - .isBefore(clock.instant().minus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS))) { + // main security index _might_ contain tokens if the tokens index has been created recently + if (false == frozenTokensIndex.indexExists() || frozenTokensIndex.getCreationTime() + .isAfter(clock.instant().minus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS))) { + if (false == frozenMainIndex.isAvailable()) { + listener.onFailure(frozenMainIndex.getUnavailableReason()); + return; + } + if (false == frozenMainIndex.isIndexUpToDate()) { + listener.onFailure(new IllegalStateException( + "Index [" + frozenMainIndex.aliasName() + "] is not on the current version. Features relying on the index" + + " will not be available until the upgrade API is run on the index")); + return; + } indicesWithTokens.add(frozenMainIndex.aliasName()); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 6fe57eda4c576..b76f00dedd03a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -160,10 +160,6 @@ public boolean isAvailable() { return this.indexState.indexAvailable; } - public boolean indexExistsButUnavailable() { - return indexExists() && false == isAvailable(); - } - public boolean isMappingUpToDate() { return this.indexState.mappingUpToDate; } From e18cc9de009deabbc4133a9615ae8d63fcf2a255 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 7 Apr 2019 22:22:38 +0300 Subject: [PATCH 34/50] Nits nits nits --- .../xpack/security/authc/TokenService.java | 100 ++++++++++-------- ...sportSamlInvalidateSessionActionTests.java | 5 +- 2 files changed, 57 insertions(+), 48 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index ebb685be697a7..20da961c6f64b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -238,7 +238,7 @@ public void createOAuth2Tokens(Authentication authentication, Authentication ori ActionListener> listener) { // the created token is compatible with the oldest node version in the cluster final Version tokenVersion = getTokenVersionCompatibility(); - // in newer versions tokens moved to a separate index + // tokens moved to a separate index in newer versions final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(tokenVersion); // the id of the created tokens ought be unguessable final String userTokenId = UUIDs.randomBase64UUID(); @@ -250,11 +250,11 @@ public void createOAuth2Tokens(Authentication authentication, Authentication ori * Create an access token and optionally a refresh token as well, based on the provided authentication and metadata, with the given * token document id. The created tokens are be stored in the security index. */ - private void createOAuth2Tokens(String userTokenId, Version version, SecurityIndexManager tokensIndex, Authentication authentication, - Authentication originatingClientAuth, Map metadata, boolean includeRefreshToken, - ActionListener> listener) { + private void createOAuth2Tokens(String userTokenId, Version tokenVersion, SecurityIndexManager tokensIndex, + Authentication authentication, Authentication originatingClientAuth, Map metadata, + boolean includeRefreshToken, ActionListener> listener) { assert userTokenId.length() == TOKEN_ID_LENGTH : "We assume token ids have a fixed length for nodes of a certain version." - + " When changing this be careful that the inferences from token length still hold."; + + " When changing the token length, be careful that the inferences about its length still hold."; ensureEnabled(); if (authentication == null) { listener.onFailure(traceLog("create token", new IllegalArgumentException("authentication must be provided"))); @@ -263,8 +263,8 @@ private void createOAuth2Tokens(String userTokenId, Version version, SecurityInd new IllegalArgumentException("originating client authentication must be provided"))); } else { final Authentication tokenAuth = new Authentication(authentication.getUser(), authentication.getAuthenticatedBy(), - authentication.getLookedUpBy(), version, AuthenticationType.TOKEN, authentication.getMetadata()); - final UserToken userToken = new UserToken(userTokenId, version, tokenAuth, getExpirationTime(), metadata); + authentication.getLookedUpBy(), tokenVersion, AuthenticationType.TOKEN, authentication.getMetadata()); + final UserToken userToken = new UserToken(userTokenId, tokenVersion, tokenAuth, getExpirationTime(), metadata); final String plainRefreshToken = includeRefreshToken ? UUIDs.randomBase64UUID() : null; final BytesReference tokenDocument = createTokenDocument(userToken, plainRefreshToken, originatingClientAuth); final String documentId = getTokenDocumentId(userToken); @@ -278,14 +278,14 @@ private void createOAuth2Tokens(String userTokenId, Version version, SecurityInd () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, indexTokenRequest, ActionListener.wrap(indexResponse -> { if (indexResponse.getResult() == Result.CREATED) { - if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { + if (tokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { final String versionedRefreshToken = plainRefreshToken != null - ? prependVersionAndEncode(version, plainRefreshToken) + ? prependVersionAndEncode(tokenVersion, plainRefreshToken) : null; listener.onResponse(new Tuple<>(userToken, versionedRefreshToken)); } else { - // prior versions are not prepended, as nodes on those versions don't expect it. Such nodes - // might exist in a mixed cluster during a rolling upgrade. + // prior versions are not version-prepended, as nodes on those versions don't expect it. + // Such nodes might exist in a mixed cluster during a rolling upgrade. listener.onResponse(new Tuple<>(userToken, plainRefreshToken)); } } else { @@ -338,10 +338,11 @@ public void getAuthenticationAndMetaData(String token, ActionListener listener) { - final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(version); + private void getUserTokenFromId(String userTokenId, Version tokenVersion, ActionListener listener) { + final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(tokenVersion); if (tokensIndex.isAvailable() == false) { logger.warn("failed to get access token [{}] because index [{}] is not available", userTokenId, tokensIndex.aliasName()); listener.onResponse(null); @@ -390,7 +391,7 @@ void getUserTokenFromId(String userTokenId, Version version, ActionListener tokenIds, SecurityIndexManager bulkRequestBuilder.add(request); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - tokensIndexManager.prepareIndexIfNeededThenExecute(ex -> listener.onFailure(traceLog("prepare security index", ex)), + tokensIndexManager.prepareIndexIfNeededThenExecute( + ex -> listener.onFailure(traceLog("prepare index [" + tokensIndexManager.aliasName() + "]", ex)), () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequestBuilder.request(), ActionListener.wrap(bulkResponse -> { ArrayList retryTokenDocIds = new ArrayList<>(); @@ -718,37 +720,38 @@ public void refreshToken(String refreshToken, ActionListener backoff, ActionListener listener) { if (refreshToken.length() == TOKEN_ID_LENGTH) { + // first check if token has the old format before the new version-prepended one logger.debug("Assuming an unversioned refresh token [{}}, generated for node versions" + " prior to the introduction of the version-header format.", refreshToken); findTokenFromRefreshToken(refreshToken, securityMainIndex, backoff, listener); } else { - final Optional> versionAndRefreshTokenTuple = tryUnpackVersionAndPayload(refreshToken); - if (versionAndRefreshTokenTuple.isPresent()) { - final Version refreshTokenVersion = versionAndRefreshTokenTuple.get().v1(); - final String unencodedRefreshToken = versionAndRefreshTokenTuple.get().v2(); + try { + final Tuple versionAndRefreshTokenTuple = unpackVersionAndPayload(refreshToken); + final Version refreshTokenVersion = versionAndRefreshTokenTuple.v1(); + final String unencodedRefreshToken = versionAndRefreshTokenTuple.v2(); if (false == refreshTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED) || unencodedRefreshToken.length() != TOKEN_ID_LENGTH) { - listener.onFailure(new IllegalArgumentException("Decoded refresh token [" + unencodedRefreshToken + "] with version [" - + refreshTokenVersion + "] is invalid.")); + logger.debug("Decoded refresh token [{}] with version [{}] is invalid.", unencodedRefreshToken, refreshTokenVersion); + listener.onFailure(malformedTokenException()); } else { findTokenFromRefreshToken(unencodedRefreshToken, securityTokensIndex, backoff, listener); } - } else { - listener.onFailure(new IllegalArgumentException("Could not decode refresh token [" + refreshToken + "].")); + } catch (IOException e) { + logger.debug("Could not decode refresh token [" + refreshToken + "].", e); + listener.onFailure(malformedTokenException()); } } } /** - * Performs an asynchronous search request for the token document that contains the {@code refreshToken} and calls the listener with the - * {@link SearchResponse}. In case of recoverable errors the {@code SearchRequest} is retried using an exponential backoff policy. This - * method requires the tokens index where the token document, pointed to by the refresh token, resides. + * Performs an asynchronous search request for the token document that contains the {@code refreshToken} and calls the {@code listener} + * with the resulting {@link SearchResponse}. In case of recoverable errors the {@code SearchRequest} is retried using an exponential + * backoff policy. This method requires the tokens index where the token document, pointed to by the refresh token, resides. */ private void findTokenFromRefreshToken(String refreshToken, SecurityIndexManager tokensIndexManager, Iterator backoff, ActionListener listener) { @@ -933,7 +936,7 @@ public void onFailure(Exception e) { } private void getSupersedingTokenDocAsyncWithRetry(RefreshTokenStatus refreshTokenStatus, Iterator backoff, - ActionListener> listener) { + ActionListener> listener) { final Consumer onFailure = ex -> listener .onFailure(traceLog("get superseding token", refreshTokenStatus.getSupersededBy(), ex)); getSupersedingTokenDocAsync(refreshTokenStatus, new ActionListener() { @@ -990,11 +993,12 @@ public void onFailure(Exception e) { private void getSupersedingTokenDocAsync(RefreshTokenStatus refreshTokenStatus, ActionListener listener) { final String supersedingDocReference = refreshTokenStatus.getSupersededBy(); if (supersedingDocReference.startsWith(securityTokensIndex.aliasName() + "|")) { + // superseding token doc is stored on the new tokens index, irrespective of where the superseded token doc resides final String supersedingDocId = supersedingDocReference.substring(securityTokensIndex.aliasName().length() + 1); getTokenDocAsync(supersedingDocId, securityTokensIndex, listener); } else { assert false == supersedingDocReference - .contains("|") : "Superseding doc reference appears to contain an alias name but shouldn't"; + .contains("|") : "The superseding doc reference appears to contain an alias name but should not"; getTokenDocAsync(supersedingDocReference, securityMainIndex, listener); } } @@ -1005,6 +1009,7 @@ private void getTokenDocAsync(String tokenDocId, SecurityIndexManager tokensInde } private Version getTokenVersionCompatibility() { + // newly minted tokens are compatible with the min node version in the cluster return clusterService.state().nodes().getMinNodeVersion(); } @@ -1210,10 +1215,10 @@ public void findActiveTokensForUser(String username, ActionListenermight contain tokens if the tokens index has not been created yet, or if it has been created recently so that there might - * still be tokens that have not yet exceeded their maximum lifetime. + * automatically created, all the onwards created tokens will be stored inside it. This function returns the list of the indices names + * that might contain tokens. Unless there are availability or version issues, the dedicated tokens index always contains tokens. The + * main security index might contain tokens if the tokens index has not been created yet, or if it has been created recently so + * that there might still be tokens that have not yet exceeded their maximum lifetime. */ private void sourceIndicesWithTokensAndRun(ActionListener> listener) { final List indicesWithTokens = new ArrayList<>(2); @@ -1359,11 +1364,11 @@ private void ensureEnabled() { } /** - * In version {@code #VERSION_TOKENS_INDEX_INTRODUCED} tokens moved into a separate index away from the other entities due to their - * ephemeral nature. But they moved "seamlessly" - without manual intervention. In this way, new tokens are created in the new index, - * while the existing ones were left in place - to be accessed from the old index - and due to be removed automatically by the {@code - * ExpiredTokenRemover} periodic job. Therefore, in general, when searching for a token we need to consider both the new and the old - * indices. + * In version {@code #VERSION_TOKENS_INDEX_INTRODUCED} security tokens were moved into a separate index, away from the other entities in + * the main security index, due to their ephemeral nature. They moved "seamlessly" - without manual user intervention. In this way, new + * tokens are created in the new index, while the existing ones were left in place - to be accessed from the old index - and due to be + * removed automatically by the {@code ExpiredTokenRemover} periodic job. Therefore, in general, when searching for a token we need to + * consider both the new and the old indices. */ private SecurityIndexManager getTokensIndexManagerForVersion(Version version) { if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { @@ -1506,20 +1511,21 @@ private static String prependVersionAndEncode(Version version, String payload) { out.writeString(payload); return new String(os.toByteArray(), StandardCharsets.UTF_8); } catch (IOException e) { - throw new RuntimeException("Unexpected exception when working with small in memory streams", e); + throw new RuntimeException("Unexpected exception when working with small in-memory streams", e); } } - private static Optional> tryUnpackVersionAndPayload(String encodedPack) { + // public for testing + /** + * Unpacks a base64 encoded pair of a version tag and String payload. + */ + public static Tuple unpackVersionAndPayload(String encodedPack) throws IOException { final byte[] bytes = encodedPack.getBytes(StandardCharsets.UTF_8); try (StreamInput in = new InputStreamStreamInput(Base64.getDecoder().wrap(new ByteArrayInputStream(bytes)), bytes.length)) { final Version version = Version.readVersion(in); in.setVersion(version); final String payload = in.readString(); - return Optional.of(new Tuple(version, payload)); - } catch (IOException | IllegalArgumentException e) { - logger.trace("Error decoding versioned String value.", e); - return Optional.empty(); + return new Tuple(version, payload); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index 7061a06fef614..bed684081b053 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -195,6 +195,9 @@ void doExecute(Action action, Request request, ActionListener Date: Sun, 7 Apr 2019 23:29:54 +0300 Subject: [PATCH 35/50] Nit --- .../org/elasticsearch/xpack/security/authc/TokenService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 20da961c6f64b..d22cd3df38406 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -721,7 +721,7 @@ public void refreshToken(String refreshToken, ActionListener backoff, ActionListener listener) { if (refreshToken.length() == TOKEN_ID_LENGTH) { From 92872ec2363e1280ad2e76ffcd28b3e2851d23c9 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 7 Apr 2019 23:49:35 +0300 Subject: [PATCH 36/50] Checkstyle --- .../action/saml/TransportSamlInvalidateSessionActionTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index bed684081b053..63c58c5ce10e8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -311,7 +311,8 @@ public void testInvalidateCorrectTokensFromLogoutRequest() throws Exception { assertThat(filter1.get(1), instanceOf(TermQueryBuilder.class)); assertThat(((TermQueryBuilder) filter1.get(1)).fieldName(), equalTo("refresh_token.token")); - assertThat(((TermQueryBuilder) filter1.get(1)).value(), equalTo(TokenService.unpackVersionAndPayload(tokenToInvalidate1.v2()).v2())); + assertThat(((TermQueryBuilder) filter1.get(1)).value(), + equalTo(TokenService.unpackVersionAndPayload(tokenToInvalidate1.v2()).v2())); assertThat(bulkRequests.size(), equalTo(4)); // 4 updates (refresh-token + access-token) // Invalidate refresh token 1 From a44f327f225716664b58246d09116e04dafc58c8 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 8 Apr 2019 11:26:59 +0300 Subject: [PATCH 37/50] Fix NativePrivilegeStoreTests after rename... --- .../xpack/security/authz/store/NativePrivilegeStoreTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index 95ae9c5351806..7f7a262131bb2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -188,7 +188,7 @@ public void testGetPrivilegesByWildcardApplicationName() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(SearchRequest.class)); SearchRequest request = (SearchRequest) requests.get(0); - assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); + assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)); final String query = Strings.toString(request.source().query()); assertThat(query, containsString("{\"bool\":{\"filter\":[{\"terms\":{\"application\":[\"yourapp\"]")); @@ -208,7 +208,7 @@ public void testGetPrivilegesByStarApplicationName() throws Exception { assertThat(requests, iterableWithSize(1)); assertThat(requests.get(0), instanceOf(SearchRequest.class)); SearchRequest request = (SearchRequest) requests.get(0); - assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)); + assertThat(request.indices(), arrayContaining(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)); final String query = Strings.toString(request.source().query()); assertThat(query, containsString("{\"exists\":{\"field\":\"application\"")); From 373c3085655c61c9a40a64deceb7b52a4e443a87 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 9 Apr 2019 14:01:51 +0300 Subject: [PATCH 38/50] TokenBackwardsCompatibilityIT --- .../xpack/security/authc/TokenService.java | 2 +- x-pack/qa/rolling-upgrade/build.gradle | 4 + .../TokenBackwardsCompatibilityIT.java | 458 ++++++++++++------ 3 files changed, 313 insertions(+), 151 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index d22cd3df38406..9a207bd0138e5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -726,7 +726,7 @@ public void refreshToken(String refreshToken, ActionListener backoff, ActionListener listener) { if (refreshToken.length() == TOKEN_ID_LENGTH) { // first check if token has the old format before the new version-prepended one - logger.debug("Assuming an unversioned refresh token [{}}, generated for node versions" + logger.debug("Assuming an unversioned refresh token [{}], generated for node versions" + " prior to the introduction of the version-header format.", refreshToken); findTokenFromRefreshToken(refreshToken, securityMainIndex, backoff, listener); } else { diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 0cdbbe71e55ba..58b6dbba6c9ac 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -121,6 +121,8 @@ for (Version version : bwcVersions.wireCompatible) { setting 'xpack.security.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' + setting 'xpack.security.authc.token.timeout', '60m' + setting 'logger.org.elasticsearch.xpack.security.authc.TokenService', 'trace' setting 'xpack.security.audit.enabled', 'true' if (project.inFipsJvm) { setting 'xpack.security.transport.ssl.key', 'testnode.pem' @@ -184,6 +186,8 @@ for (Version version : bwcVersions.wireCompatible) { setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' + setting 'xpack.security.authc.token.timeout', '60m' + setting 'logger.org.elasticsearch.xpack.security.authc.TokenService', 'trace' if (project.inFipsJvm) { setting 'xpack.security.transport.ssl.key', 'testnode.pem' setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index 0dca61eba5721..4564a9b1f51c8 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -14,197 +14,355 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.test.rest.yaml.ObjectPath; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase { - public void testGeneratingTokenInOldCluster() throws Exception { + private Map clientsByVersion = null; + + @Before + private void collateClientsByVersion() throws IOException { + clientsByVersion = getRestClientByVersion(); + } + + @After + private void closeClientsByVersion() throws IOException { + for (RestClient client : clientsByVersion.values()) { + client.close(); + } + clientsByVersion = null; + } + + public void testGeneratingTokensInOldCluster() throws Exception { assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD); - // Create a couple of tokens and store them in the token_backwards_compatibility_it index to be used for tests in the mixed/upgraded - // clusters - Request createTokenRequest = new Request("POST", "/_security/oauth2/token"); - createTokenRequest.setJsonEntity( - "{\n" + - " \"username\": \"test_user\",\n" + - " \"password\": \"x-pack-test-password\",\n" + - " \"grant_type\": \"password\"\n" + - "}"); - Response response = client().performRequest(createTokenRequest); - assertOK(response); - Map responseMap = entityAsMap(response); - String token = (String) responseMap.get("access_token"); - assertNotNull(token); - assertTokenWorks(token); + // Creates two access and refresh tokens and stores them in the token_backwards_compatibility_it index to be used for tests in the + // mixed/upgraded clusters + Map responseMap = createTokens(client(), "test_user", "x-pack-test-password"); + String accessToken = (String) responseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + String refreshToken = (String) responseMap.get("refresh_token"); + assertNotNull(refreshToken); - Request indexRequest1 = new Request("PUT", "token_backwards_compatibility_it/_doc/old_cluster_token1"); - indexRequest1.setJsonEntity( - "{\n" + - " \"token\": \"" + token + "\"\n" + - "}"); - Response indexResponse1 = client().performRequest(indexRequest1); - assertOK(indexResponse1); - Request createSecondTokenRequest = new Request("POST", "/_security/oauth2/token"); - createSecondTokenRequest.setEntity(createTokenRequest.getEntity()); - response = client().performRequest(createSecondTokenRequest); - responseMap = entityAsMap(response); - token = (String) responseMap.get("access_token"); - assertNotNull(token); - assertTokenWorks(token); - Request indexRequest2 = new Request("PUT", "token_backwards_compatibility_it/_doc/old_cluster_token2"); - indexRequest2.setJsonEntity( - "{\n" + - " \"token\": \"" + token + "\"\n" + - "}"); - Response indexResponse2 = client().performRequest(indexRequest2); - assertOK(indexResponse2); + storeTokens(client(), 1, accessToken, refreshToken); + + responseMap = createTokens(client(), "test_user", "x-pack-test-password"); + accessToken = (String) responseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + refreshToken = (String) responseMap.get("refresh_token"); + assertNotNull(refreshToken); + + storeTokens(client(), 2, accessToken, refreshToken); } - public void testTokenWorksInMixedCluster() throws Exception { + public void testRefreshingTokensInOldCluster() throws Exception { + assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD); + // Creates another access and refresh tokens and tries to use the refresh tokens several times + Map responseMap = createTokens(client(), "test_user", "x-pack-test-password"); + String accessToken = (String) responseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + String refreshToken = (String) responseMap.get("refresh_token"); + assertNotNull(refreshToken); + + storeTokens(client(), 3, accessToken, refreshToken); + + // refresh the new token + Map refreshResponseMap = refreshToken(client(), refreshToken); + String refreshedAccessToken = (String) refreshResponseMap.get("access_token"); + String refreshedRefreshToken = (String) refreshResponseMap.get("refresh_token"); + assertNotNull(refreshedAccessToken); + assertNotNull(refreshedRefreshToken); + assertAccessTokenWorks(refreshedAccessToken); + // assert previous access token still works + assertAccessTokenWorks(accessToken); + + storeTokens(client(), 4, refreshedAccessToken, refreshedRefreshToken); + } + + public void testInvalidatingTokensInOldCluster() throws Exception { + assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD); + // Creates another access and refresh tokens and tries to use the refresh tokens several times + Map responseMap = createTokens(client(), "test_user", "x-pack-test-password"); + String accessToken = (String) responseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + String refreshToken = (String) responseMap.get("refresh_token"); + assertNotNull(refreshToken); + + storeTokens(client(), 5, accessToken, refreshToken); + + // invalidate access token + invalidateAccessToken(client(), accessToken); + assertAccessTokenDoesNotWork(accessToken); + // invalidate refresh token + invalidateRefreshToken(client(), refreshToken); + assertRefreshTokenInvalidated(refreshToken); + } + + public void testAccessTokensWorkInMixedCluster() throws Exception { // Verify that an old token continues to work during all stages of the rolling upgrade assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); - Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token1"); - Response getResponse = client().performRequest(getRequest); - assertOK(getResponse); - Map source = (Map) entityAsMap(getResponse).get("_source"); - assertTokenWorks((String) source.get("token")); + for (int tokenIdx : Arrays.asList(1, 3, 4)) { // 2 is invalidated in another mixed-cluster test, 5 is invalidated in the old cluster + Map source = retrieveStoredTokens(client(), tokenIdx); + assertAccessTokenWorks((String) source.get("token")); + } } - public void testInvalidatingTokenInMixedCluster() throws Exception { - // Verify that we can invalidate a token in a mixed cluster + public void testTokensStayInvalidatedInMixedCluster() throws Exception { + // Verify that an old token continues to work during all stages of the rolling upgrade assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); - Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token2"); - Response getResponse = client().performRequest(getRequest); - assertOK(getResponse); - Map source = (Map) entityAsMap(getResponse).get("_source"); - String token = (String) source.get("token"); - // The token might be already invalidated by running testInvalidatingTokenInMixedCluster in a previous stage - // we don't try to assert it works before invalidating. This case is handled by testTokenWorksInMixedCluster - Request invalidateRequest = new Request("DELETE", "/_security/oauth2/token"); - invalidateRequest.setJsonEntity("{\"token\": \"" + token + "\"}"); - invalidateRequest.addParameter("error_trace", "true"); - client().performRequest(invalidateRequest); - assertTokenDoesNotWork(token); + Map source = retrieveStoredTokens(client(), 5); + assertAccessTokenDoesNotWork((String) source.get("token")); + assertRefreshTokenInvalidated((String) source.get("refresh_token")); } - public void testMixedClusterWithUpgradedMaster() throws Exception { + public void testGeneratingTokensInMixedCluster() throws Exception { assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); - assumeTrue("the master must be on the latest version before we can write", isMasterOnLatestVersion()); - - // create token and refresh on version that supports it - Request createTokenRequest = new Request("POST", "/_security/oauth2/token"); - createTokenRequest.setJsonEntity( - "{\n" + - " \"username\": \"test_user\",\n" + - " \"password\": \"x-pack-test-password\",\n" + - " \"grant_type\": \"password\"\n" + - "}"); - try (RestClient client = getRestClientForCurrentVersionNodesOnly()) { - Response response = client.performRequest(createTokenRequest); - Map responseMap = entityAsMap(response); + // Creates two access and refresh tokens and stores them in the token_backwards_compatibility_it index to be used for tests in the + // mixed/upgraded clusters + int generatedTokenIdxDuringMixed = 10; + assert clientsByVersion.size() == 2 : "A rolling upgrade has only two versions of nodes, found: " + clientsByVersion.keySet(); + for (RestClient client : clientsByVersion.values()) { + Map responseMap = createTokens(client, "test_user", "x-pack-test-password"); String accessToken = (String) responseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); String refreshToken = (String) responseMap.get("refresh_token"); + assertNotNull(refreshToken); + + storeTokens(client(), generatedTokenIdxDuringMixed++, accessToken, refreshToken); + + responseMap = createTokens(client, "test_user", "x-pack-test-password"); + accessToken = (String) responseMap.get("access_token"); assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + refreshToken = (String) responseMap.get("refresh_token"); assertNotNull(refreshToken); - assertTokenWorks(accessToken); - Request tokenRefreshRequest = new Request("POST", "/_security/oauth2/token"); - tokenRefreshRequest.setJsonEntity( - "{\n" + - " \"refresh_token\": \"" + refreshToken + "\",\n" + - " \"grant_type\": \"refresh_token\"\n" + - "}"); - response = client.performRequest(tokenRefreshRequest); - responseMap = entityAsMap(response); - String updatedAccessToken = (String) responseMap.get("access_token"); - String updatedRefreshToken = (String) responseMap.get("refresh_token"); - assertNotNull(updatedAccessToken); - assertNotNull(updatedRefreshToken); - assertTokenWorks(updatedAccessToken); - assertTokenWorks(accessToken); - assertNotEquals(accessToken, updatedAccessToken); - assertNotEquals(refreshToken, updatedRefreshToken); - // Invalidate the new access token and ensure that it no longer works - Request invalidateTokenRequest = new Request("DELETE", "/_security/oauth2/token"); - invalidateTokenRequest.setJsonEntity( - "{\n" + - " \"token\": \"" + updatedAccessToken + "\"\n" + - "}"); - Response invalidateTokenResponse = client.performRequest(invalidateTokenRequest); - assertOK(invalidateTokenResponse); - assertTokenDoesNotWork(updatedAccessToken); + storeTokens(client(), generatedTokenIdxDuringMixed++, accessToken, refreshToken); + } + } + + public void testRefreshingTokensInMixedCluster() throws Exception { + // verify new nodes can refresh tokens from the old cluster + assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); + assert clientsByVersion.size() == 2 : "A rolling upgrade has only two versions of nodes, found: " + clientsByVersion.keySet(); + for (RestClient client1 : clientsByVersion.values()) { + Map responseMap = createTokens(client1, "test_user", "x-pack-test-password"); + String accessToken = (String) responseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + String refreshToken = (String) responseMap.get("refresh_token"); + assertNotNull(refreshToken); + for (RestClient client2 : clientsByVersion.values()) { + responseMap = refreshToken(client2, refreshToken); + accessToken = (String) responseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + refreshToken = (String) responseMap.get("refresh_token"); + assertNotNull(refreshToken); + } } } - public void testUpgradedCluster() throws Exception { + public void testInvalidatingTokensInMixedCluster() throws Exception { + // Verify that we can invalidate an access and refresh token in a mixed cluster + assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); + Map source = retrieveStoredTokens(client(), 2); + String accessToken = (String) source.get("token"); + String refreshToken = (String) source.get("refresh_token"); + // The token might be already invalidated by running testInvalidatingTokenInMixedCluster in a previous stage + // we don't try to assert it works before invalidating. This case is handled by testTokenWorksInMixedCluster + invalidateAccessToken(client(), accessToken); + assertAccessTokenDoesNotWork(accessToken); + // invalidate refresh token + invalidateRefreshToken(client(), refreshToken); + assertRefreshTokenInvalidated(refreshToken); + } + + public void testTokensStayInvalidatedInUpgradedCluster() throws Exception { assumeTrue("this test should only run against the upgraded cluster", CLUSTER_TYPE == ClusterType.UPGRADED); + for (int tokenIdx : Arrays.asList(2, 5)) { + Map source = retrieveStoredTokens(client(), tokenIdx); + assertAccessTokenDoesNotWork((String) source.get("token")); + assertRefreshTokenInvalidated((String) source.get("refresh_token")); + } + } - // Use an old token to authenticate, then invalidate it and verify that it can no longer be used - Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token1"); - Response getResponse = client().performRequest(getRequest); - assertOK(getResponse); - Map source = (Map) entityAsMap(getResponse).get("_source"); - final String token = (String) source.get("token"); + public void testAccessTokensWorkInUpgradedCluster() throws Exception { + assumeTrue("this test should only run against the upgraded cluster", CLUSTER_TYPE == ClusterType.UPGRADED); + for (int tokenIdx : Arrays.asList(3, 4, 10, 12)) { + Map source = retrieveStoredTokens(client(), tokenIdx); + assertAccessTokenWorks((String) source.get("token")); + } + } - Request invalidateRequest = new Request("DELETE", "/_security/oauth2/token"); - invalidateRequest.setJsonEntity("{\"token\": \"" + token + "\"}"); - invalidateRequest.addParameter("error_trace", "true"); - Response invalidationResponse = client().performRequest(invalidateRequest); - assertOK(invalidationResponse); - assertTokenDoesNotWork(token); - } - - private void assertTokenWorks(String token) throws IOException { - Request request = new Request("GET", "/_security/_authenticate"); - RequestOptions.Builder options = request.getOptions().toBuilder(); - options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token); - request.setOptions(options); - Response authenticateResponse = client().performRequest(request); - assertOK(authenticateResponse); - assertEquals("test_user", entityAsMap(authenticateResponse).get("username")); - } - - private void assertTokenDoesNotWork(String token) { - Request request = new Request("GET", "/_security/_authenticate"); - RequestOptions.Builder options = request.getOptions().toBuilder(); - options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token); - request.setOptions(options); - ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); - assertEquals(401, e.getResponse().getStatusLine().getStatusCode()); - Response response = e.getResponse(); - assertEquals("Bearer realm=\"security\", error=\"invalid_token\", error_description=\"The access token expired\"", - response.getHeader("WWW-Authenticate")); - } - - private boolean isMasterOnLatestVersion() throws Exception { - Response response = client().performRequest(new Request("GET", "_cluster/state")); - assertOK(response); - final String masterNodeId = ObjectPath.createFromResponse(response).evaluate("master_node"); - response = client().performRequest(new Request("GET", "_nodes")); - assertOK(response); - ObjectPath objectPath = ObjectPath.createFromResponse(response); - logger.info("Master node is on version: " + objectPath.evaluate("nodes." + masterNodeId + ".version")); - return Version.CURRENT.equals(Version.fromString(objectPath.evaluate("nodes." + masterNodeId + ".version"))); + public void testGeneratingTokensInUpgradedCluster() throws Exception { + assumeTrue("this test should only run against the upgraded cluster", CLUSTER_TYPE == ClusterType.UPGRADED); + Map responseMap = createTokens(client(), "test_user", "x-pack-test-password"); + String accessToken = (String) responseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + String refreshToken = (String) responseMap.get("refresh_token"); + assertNotNull(refreshToken); + } + + public void testRefreshingTokensInUpgradedCluster() throws Exception { + assumeTrue("this test should only run against the upgraded cluster", CLUSTER_TYPE == ClusterType.UPGRADED); + for (int tokenIdx : Arrays.asList(4, 10, 12)) { + Map source = retrieveStoredTokens(client(), tokenIdx); + Map refreshedResponseMap = refreshToken(client(), (String) source.get("refresh_token")); + String accessToken = (String) refreshedResponseMap.get("access_token"); + assertNotNull(accessToken); + assertAccessTokenWorks(accessToken); + String refreshToken = (String) refreshedResponseMap.get("refresh_token"); + assertNotNull(refreshToken); + } + } + + public void testInvalidatingTokensInUpgradedCluster() throws Exception { + assumeTrue("this test should only run against the upgraded cluster", CLUSTER_TYPE == ClusterType.UPGRADED); + for (int tokenIdx : Arrays.asList(1, 11, 13)) { + Map source = retrieveStoredTokens(client(), tokenIdx); + String accessToken = (String) source.get("token"); + String refreshToken = (String) source.get("refresh_token"); + // invalidate access token + invalidateAccessToken(client(), accessToken); + assertAccessTokenDoesNotWork(accessToken); + // invalidate refresh token + invalidateRefreshToken(client(), refreshToken); + assertRefreshTokenInvalidated(refreshToken); + } + } + + private void assertAccessTokenWorks(String token) throws IOException { + for (RestClient client : clientsByVersion.values()) { + Request request = new Request("GET", "/_security/_authenticate"); + RequestOptions.Builder options = request.getOptions().toBuilder(); + options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token); + request.setOptions(options); + Response authenticateResponse = client.performRequest(request); + assertOK(authenticateResponse); + assertEquals("test_user", entityAsMap(authenticateResponse).get("username")); + } + } + + private void assertAccessTokenDoesNotWork(String token) throws IOException { + for (RestClient client : clientsByVersion.values()) { + Request request = new Request("GET", "/_security/_authenticate"); + RequestOptions.Builder options = request.getOptions().toBuilder(); + options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token); + request.setOptions(options); + ResponseException e = expectThrows(ResponseException.class, () -> client.performRequest(request)); + assertEquals(401, e.getResponse().getStatusLine().getStatusCode()); + Response response = e.getResponse(); + assertEquals("Bearer realm=\"security\", error=\"invalid_token\", error_description=\"The access token expired\"", + response.getHeader("WWW-Authenticate")); + } + } + + private void assertRefreshTokenInvalidated(String refreshToken) throws IOException { + for (RestClient client : clientsByVersion.values()) { + Request refreshTokenRequest = new Request("POST", "/_security/oauth2/token"); + refreshTokenRequest.setJsonEntity( + "{\n" + + " \"refresh_token\": \"" + refreshToken + "\",\n" + + " \"grant_type\": \"refresh_token\"\n" + + "}"); + ResponseException e = expectThrows(ResponseException.class, () -> client.performRequest(refreshTokenRequest)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + Response response = e.getResponse(); + Map responseMap = entityAsMap(response); + assertEquals("invalid_grant", responseMap.get("error")); + assertEquals("token has been invalidated", responseMap.get("error_description")); + } } - private RestClient getRestClientForCurrentVersionNodesOnly() throws IOException { + private Map getRestClientByVersion() throws IOException { Response response = client().performRequest(new Request("GET", "_nodes")); assertOK(response); ObjectPath objectPath = ObjectPath.createFromResponse(response); Map nodesAsMap = objectPath.evaluate("nodes"); - List hosts = new ArrayList<>(); + Map> hostsByVersion = new HashMap<>(); for (Map.Entry entry : nodesAsMap.entrySet()) { Map nodeDetails = (Map) entry.getValue(); Version version = Version.fromString((String) nodeDetails.get("version")); - if (Version.CURRENT.equals(version)) { - Map httpInfo = (Map) nodeDetails.get("http"); - hosts.add(HttpHost.create((String) httpInfo.get("publish_address"))); - } + Map httpInfo = (Map) nodeDetails.get("http"); + hostsByVersion.computeIfAbsent(version, k -> new ArrayList<>()).add(HttpHost.create((String) httpInfo.get("publish_address"))); + } + Map clientsByVersion = new HashMap<>(); + for (Map.Entry> entry : hostsByVersion.entrySet()) { + clientsByVersion.put(entry.getKey(), buildClient(restClientSettings(), entry.getValue().toArray(new HttpHost[0]))); } + return clientsByVersion; + } - return buildClient(restClientSettings(), hosts.toArray(new HttpHost[0])); + private Map createTokens(RestClient client, String username, String password) throws IOException { + final Request createTokenRequest = new Request("POST", "/_security/oauth2/token"); + createTokenRequest.setJsonEntity( + "{\n" + + " \"username\": \"" + username + "\",\n" + + " \"password\": \"" + password + "\",\n" + + " \"grant_type\": \"password\"\n" + + "}"); + Response response = client().performRequest(createTokenRequest); + assertOK(response); + return entityAsMap(response); + } + + private void storeTokens(RestClient client, int idx, String accessToken, String refreshToken) throws IOException { + final Request indexRequest = new Request("PUT", "token_backwards_compatibility_it/_doc/old_cluster_token" + idx); + indexRequest.setJsonEntity( + "{\n" + + " \"token\": \"" + accessToken + "\",\n" + + " \"refresh_token\": \"" + refreshToken + "\"\n" + + "}"); + Response indexResponse1 = client.performRequest(indexRequest); + assertOK(indexResponse1); + } + + private Map retrieveStoredTokens(RestClient client, int tokenIdx) throws IOException { + Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token" + tokenIdx); + Response getResponse = client().performRequest(getRequest); + assertOK(getResponse); + return (Map) entityAsMap(getResponse).get("_source"); + } + + private Map refreshToken(RestClient client, String refreshToken) throws IOException { + final Request refreshTokenRequest = new Request("POST", "/_security/oauth2/token"); + refreshTokenRequest.setJsonEntity( + "{\n" + + " \"refresh_token\": \"" + refreshToken + "\",\n" + + " \"grant_type\": \"refresh_token\"\n" + + "}"); + Response refreshResponse = client.performRequest(refreshTokenRequest); + assertOK(refreshResponse); + return entityAsMap(refreshResponse); + } + + private void invalidateAccessToken(RestClient client, String accessToken) throws IOException { + Request invalidateRequest = new Request("DELETE", "/_security/oauth2/token"); + invalidateRequest.setJsonEntity("{\"token\": \"" + accessToken + "\"}"); + invalidateRequest.addParameter("error_trace", "true"); + Response invalidateResponse = client.performRequest(invalidateRequest); + assertOK(invalidateResponse); + } + + private void invalidateRefreshToken(RestClient client, String refreshToken) throws IOException { + Request invalidateRequest = new Request("DELETE", "/_security/oauth2/token"); + invalidateRequest.setJsonEntity("{\"refresh_token\": \"" + refreshToken + "\"}"); + invalidateRequest.addParameter("error_trace", "true"); + Response invalidateResponse = client.performRequest(invalidateRequest); + assertOK(invalidateResponse); } } From a850f760cf613c8ff5f08b9e54e72fefb34d9fed Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 9 Apr 2019 16:48:34 +0300 Subject: [PATCH 39/50] Golden TokenBackwardsCompatibilityIT --- .../TokenBackwardsCompatibilityIT.java | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index 4564a9b1f51c8..cca3297b0ce40 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -20,25 +20,37 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase { - private Map clientsByVersion = null; + private Collection twoClients = null; @Before private void collateClientsByVersion() throws IOException { - clientsByVersion = getRestClientByVersion(); + Map clientsByVersion = getRestClientByVersion(); + if (clientsByVersion.size() == 2) { + // usual case, clients have different versions + twoClients = clientsByVersion.values(); + } else { + assert clientsByVersion.size() == 1 : "A rolling upgrade has a maximum of two distinct node versions, found: " + + clientsByVersion.keySet(); + // tests assumes exactly two clients to simplify some logic + twoClients = new ArrayList<>(); + twoClients.add(clientsByVersion.values().iterator().next()); + twoClients.add(clientsByVersion.values().iterator().next()); + } } @After private void closeClientsByVersion() throws IOException { - for (RestClient client : clientsByVersion.values()) { + for (RestClient client : twoClients) { client.close(); } - clientsByVersion = null; + twoClients = null; } public void testGeneratingTokensInOldCluster() throws Exception { @@ -131,8 +143,7 @@ public void testGeneratingTokensInMixedCluster() throws Exception { // Creates two access and refresh tokens and stores them in the token_backwards_compatibility_it index to be used for tests in the // mixed/upgraded clusters int generatedTokenIdxDuringMixed = 10; - assert clientsByVersion.size() == 2 : "A rolling upgrade has only two versions of nodes, found: " + clientsByVersion.keySet(); - for (RestClient client : clientsByVersion.values()) { + for (RestClient client : twoClients) { Map responseMap = createTokens(client, "test_user", "x-pack-test-password"); String accessToken = (String) responseMap.get("access_token"); assertNotNull(accessToken); @@ -156,15 +167,14 @@ public void testGeneratingTokensInMixedCluster() throws Exception { public void testRefreshingTokensInMixedCluster() throws Exception { // verify new nodes can refresh tokens from the old cluster assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); - assert clientsByVersion.size() == 2 : "A rolling upgrade has only two versions of nodes, found: " + clientsByVersion.keySet(); - for (RestClient client1 : clientsByVersion.values()) { + for (RestClient client1 : twoClients) { Map responseMap = createTokens(client1, "test_user", "x-pack-test-password"); String accessToken = (String) responseMap.get("access_token"); assertNotNull(accessToken); assertAccessTokenWorks(accessToken); String refreshToken = (String) responseMap.get("refresh_token"); assertNotNull(refreshToken); - for (RestClient client2 : clientsByVersion.values()) { + for (RestClient client2 : twoClients) { responseMap = refreshToken(client2, refreshToken); accessToken = (String) responseMap.get("access_token"); assertNotNull(accessToken); @@ -246,7 +256,7 @@ public void testInvalidatingTokensInUpgradedCluster() throws Exception { } private void assertAccessTokenWorks(String token) throws IOException { - for (RestClient client : clientsByVersion.values()) { + for (RestClient client : twoClients) { Request request = new Request("GET", "/_security/_authenticate"); RequestOptions.Builder options = request.getOptions().toBuilder(); options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token); @@ -258,7 +268,7 @@ private void assertAccessTokenWorks(String token) throws IOException { } private void assertAccessTokenDoesNotWork(String token) throws IOException { - for (RestClient client : clientsByVersion.values()) { + for (RestClient client : twoClients) { Request request = new Request("GET", "/_security/_authenticate"); RequestOptions.Builder options = request.getOptions().toBuilder(); options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token); @@ -272,7 +282,7 @@ private void assertAccessTokenDoesNotWork(String token) throws IOException { } private void assertRefreshTokenInvalidated(String refreshToken) throws IOException { - for (RestClient client : clientsByVersion.values()) { + for (RestClient client : twoClients) { Request refreshTokenRequest = new Request("POST", "/_security/oauth2/token"); refreshTokenRequest.setJsonEntity( "{\n" + From 526936d57365cd494ad9be7a4e699097b2104ef5 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 9 Apr 2019 17:34:52 +0300 Subject: [PATCH 40/50] Token Index Visibility test --- .../hidden-index/13_security-tokens_read.yml | 161 ++++++++++++++++++ .../14_security-tokens-7_read.yml | 161 ++++++++++++++++++ .../test/old_cluster/50_token_auth.yml | 4 +- .../test/upgraded_cluster/50_token_auth.yml | 2 +- 4 files changed, 326 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/security/hidden-index/13_security-tokens_read.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/security/hidden-index/14_security-tokens-7_read.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/hidden-index/13_security-tokens_read.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/hidden-index/13_security-tokens_read.yml new file mode 100644 index 0000000000000..183731e1ba839 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/hidden-index/13_security-tokens_read.yml @@ -0,0 +1,161 @@ +--- +setup: + - skip: + features: headers + + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_role: + name: "all_access" + body: > + { + "cluster": [ "all" ], + "indices": [ + { "names": ["*"], "privileges": ["all"] } + ] + } + + - do: + security.put_user: + username: "test_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "all_access" ], + "full_name" : "user with all possible privileges (but not superuser)" + } + +--- +teardown: + - do: + security.delete_user: + username: "test_user" + ignore: 404 + + - do: + security.delete_role: + name: "all_access" + ignore: 404 + +--- +"Test get security tokens index metadata": + + - do: + security.get_token: + body: + grant_type: "password" + username: "test_user" + password: "x-pack-test-password" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - match: { expires_in: 1200 } + - is_false: scope + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.get: + index: ".security-tokens" + + - do: + catch: forbidden + headers: + Authorization: Bearer ${token} + indices.get: + index: ".security-tokens" + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.get: + index: ".secu*rity*tokens" + - length: { $body: 0 } + + - do: + headers: + Authorization: Bearer ${token} + indices.get: + index: ".secu*rity*tokens" + - length: { $body: 0 } + +--- +"Test get security document": + + - do: + security.get_token: + body: + grant_type: "password" + username: "test_user" + password: "x-pack-test-password" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - match: { expires_in: 1200 } + - is_false: scope + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + get: + index: ".security-tokens" + id: token_${token} + + - do: + catch: forbidden + headers: + Authorization: Bearer ${token} + get: + index: ".security-tokens" + id: token_${token} + +--- +"Test search security tokens index": + + - do: + security.get_token: + body: + grant_type: "password" + username: "test_user" + password: "x-pack-test-password" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - match: { expires_in: 1200 } + - is_false: scope + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: ".security-tokens" + + - do: + catch: forbidden + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: ".security-tokens" + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: ".secu*rity*tokens" + - match: { hits.total: 0 } + + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: ".secu*rity*tokens" + - match: { hits.total: 0 } + diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/hidden-index/14_security-tokens-7_read.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/hidden-index/14_security-tokens-7_read.yml new file mode 100644 index 0000000000000..97cff4ba98035 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/hidden-index/14_security-tokens-7_read.yml @@ -0,0 +1,161 @@ +--- +setup: + - skip: + features: headers + + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_role: + name: "all_access" + body: > + { + "cluster": [ "all" ], + "indices": [ + { "names": ["*"], "privileges": ["all"] } + ] + } + + - do: + security.put_user: + username: "test_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "all_access" ], + "full_name" : "user with all possible privileges (but not superuser)" + } + +--- +teardown: + - do: + security.delete_user: + username: "test_user" + ignore: 404 + + - do: + security.delete_role: + name: "all_access" + ignore: 404 + +--- +"Test get security tokens index metadata": + + - do: + security.get_token: + body: + grant_type: "password" + username: "test_user" + password: "x-pack-test-password" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - match: { expires_in: 1200 } + - is_false: scope + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.get: + index: ".security-tokens-7" + + - do: + catch: forbidden + headers: + Authorization: Bearer ${token} + indices.get: + index: ".security-tokens-7" + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.get: + index: ".secu*rity*tokens-7" + - length: { $body: 0 } + + - do: + headers: + Authorization: Bearer ${token} + indices.get: + index: ".secu*rity*tokens-7" + - length: { $body: 0 } + +--- +"Test get security document": + + - do: + security.get_token: + body: + grant_type: "password" + username: "test_user" + password: "x-pack-test-password" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - match: { expires_in: 1200 } + - is_false: scope + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + get: + index: ".security-tokens-7" + id: token_${token} + + - do: + catch: forbidden + headers: + Authorization: Bearer ${token} + get: + index: ".security-tokens-7" + id: token_${token} + +--- +"Test search security tokens index": + + - do: + security.get_token: + body: + grant_type: "password" + username: "test_user" + password: "x-pack-test-password" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - match: { expires_in: 1200 } + - is_false: scope + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: ".security-tokens-7" + + - do: + catch: forbidden + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: ".security-tokens-7" + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + search: + rest_total_hits_as_int: true + index: ".secu*rity*tokens-7" + - match: { hits.total: 0 } + + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: ".secu*rity*tokens-7" + - match: { hits.total: 0 } + diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/50_token_auth.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/50_token_auth.yml index 02fa0f31ce37d..49ce61c1fe615 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/50_token_auth.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/50_token_auth.yml @@ -27,7 +27,9 @@ - match: { type: "Bearer" } - is_true: access_token - set: { access_token: token } - - match: { expires_in: 1200 } + - is_true: refresh_token + - set: { refresh_token: refresh_token } + - match: { expires_in: 3600 } - is_false: scope - do: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/50_token_auth.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/50_token_auth.yml index f892c9f2f9833..398f50f3c6e79 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/50_token_auth.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/50_token_auth.yml @@ -2,7 +2,7 @@ "Get the indexed token and use if to authenticate": - skip: features: headers - + - do: get: index: token_index From 17e741568a69c57f7942da6643f96e2c29b2527d Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 9 Apr 2019 20:44:19 +0300 Subject: [PATCH 41/50] UpgradeClusterClientYamlTestSuiteIT refresh tokens --- .../test/mixed_cluster/50_token_auth.yml | 140 +++++++++++++++++- .../test/old_cluster/50_token_auth.yml | 54 ++++++- .../test/upgraded_cluster/50_token_auth.yml | 57 ++++++- 3 files changed, 237 insertions(+), 14 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml index aae5f30859743..5e3fce8ef6fba 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml @@ -3,14 +3,17 @@ - skip: features: headers + - do: + cluster.health: + wait_for_status: yellow + - do: get: index: token_index - type: doc id: "6" - match: { _index: token_index } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "6" } - is_true: _source.token - set: { _source.token : token } @@ -24,6 +27,125 @@ - match: { roles.0: "superuser" } - match: { full_name: "Token User" } + # call three times because the client rotates the nodes + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: token_index + + - match: { hits.total: 8 } + + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: token_index + + - match: { hits.total: 8 } + + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: token_index + + - match: { hits.total: 8 } + +--- +"Get the indexed refreshed access token and use if to authenticate": + - skip: + features: headers + + - do: + get: + index: token_index + id: "7" + + - match: { _index: token_index } + - match: { _type: _doc } + - match: { _id: "7" } + - is_true: _source.token + - set: { _source.token : token } + + - do: + headers: + Authorization: Bearer ${token} + security.authenticate: {} + + - match: { username: "token_user" } + - match: { roles.0: "superuser" } + - match: { full_name: "Token User" } + + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: token_index + + - match: { hits.total: 8 } + + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: token_index + + - match: { hits.total: 8 } + + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: token_index + + - match: { hits.total: 8 } + +--- +"Get the indexed refresh token and use if to get another access token and authenticate": + - skip: + features: headers + + - do: + get: + index: token_index + id: "8" + + - match: { _index: token_index } + - match: { _type: _doc } + - match: { _id: "8" } + - is_true: _source.token + - set: { _source.token : refresh_token } + + - do: + security.get_token: + body: + grant_type: "refresh_token" + refresh_token: "${refresh_token}" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - is_true: refresh_token + - set: { refresh_token: refresh_token } + - match: { expires_in: 3600 } + - is_false: scope + + - do: + headers: + Authorization: Bearer ${token} + security.authenticate: {} + + - match: { username: "token_user" } + - match: { roles.0: "superuser" } + - match: { full_name: "Token User" } + - do: headers: Authorization: Bearer ${token} @@ -31,7 +153,7 @@ rest_total_hits_as_int: true index: token_index - - match: { hits.total: 6 } + - match: { hits.total: 8 } - do: headers: @@ -40,7 +162,7 @@ rest_total_hits_as_int: true index: token_index - - match: { hits.total: 6 } + - match: { hits.total: 8 } - do: headers: @@ -49,5 +171,13 @@ rest_total_hits_as_int: true index: token_index - - match: { hits.total: 6 } + - match: { hits.total: 8 } + # overwrite the used refresh token with the new one + - do: + headers: + Authorization: Bearer ${token} + index: + index: token_index + id: "8" + body: { "token" : "${refresh_token}"} diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/50_token_auth.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/50_token_auth.yml index 49ce61c1fe615..e4d0eb8757f85 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/50_token_auth.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/50_token_auth.yml @@ -56,15 +56,15 @@ bulk: refresh: true body: - - '{"index": {"_index": "token_index", "_type": "doc", "_id" : "1"}}' + - '{"index": {"_index": "token_index", "_type": "_doc", "_id" : "1"}}' - '{"f1": "v1_old", "f2": 0}' - - '{"index": {"_index": "token_index", "_type": "doc", "_id" : "2"}}' + - '{"index": {"_index": "token_index", "_type": "_doc", "_id" : "2"}}' - '{"f1": "v2_old", "f2": 1}' - - '{"index": {"_index": "token_index", "_type": "doc", "_id" : "3"}}' + - '{"index": {"_index": "token_index", "_type": "_doc", "_id" : "3"}}' - '{"f1": "v3_old", "f2": 2}' - - '{"index": {"_index": "token_index", "_type": "doc", "_id" : "4"}}' + - '{"index": {"_index": "token_index", "_type": "_doc", "_id" : "4"}}' - '{"f1": "v4_old", "f2": 3}' - - '{"index": {"_index": "token_index", "_type": "doc", "_id" : "5"}}' + - '{"index": {"_index": "token_index", "_type": "_doc", "_id" : "5"}}' - '{"f1": "v5_old", "f2": 4}' - do: @@ -83,6 +83,48 @@ Authorization: Bearer ${token} index: index: token_index - type: doc id: "6" body: { "token" : "${token}"} + + # refresh token and store it as well + - do: + security.get_token: + body: + grant_type: "refresh_token" + refresh_token: "${refresh_token}" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: refreshed_access_token } + - is_true: refresh_token + - set: { refresh_token: refreshed_refresh_token } + - match: { expires_in: 3600 } + - is_false: scope + + # test refresh token (use it) + - do: + headers: + Authorization: Bearer ${refreshed_access_token} + security.authenticate: {} + + - match: { username: "token_user" } + - match: { roles.0: "superuser" } + - match: { full_name: "Token User" } + + # store the new refreshed access token + - do: + headers: + Authorization: Bearer ${refreshed_access_token} + index: + index: token_index + id: "7" + body: { "token" : "${refreshed_access_token}"} + + # store the refresh token + - do: + headers: + Authorization: Bearer ${refreshed_access_token} + index: + index: token_index + id: "8" + body: { "token" : "${refreshed_refresh_token}"} diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/50_token_auth.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/50_token_auth.yml index 398f50f3c6e79..430f94c1064d6 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/50_token_auth.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/50_token_auth.yml @@ -3,14 +3,17 @@ - skip: features: headers + - do: + cluster.health: + wait_for_status: yellow + - do: get: index: token_index - type: doc id: "6" - match: { _index: token_index } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "6" } - is_true: _source.token - set: { _source.token : token } @@ -31,7 +34,7 @@ rest_total_hits_as_int: true index: token_index - - match: { hits.total: 6 } + - match: { hits.total: 8 } # counter example that we are really checking this - do: @@ -41,3 +44,51 @@ search: rest_total_hits_as_int: true index: token_index + +--- +"Get the indexed refresh token and use if to get another access token and authenticate": + - skip: + features: headers + + - do: + get: + index: token_index + id: "8" + + - match: { _index: token_index } + - match: { _type: _doc } + - match: { _id: "8" } + - is_true: _source.token + - set: { _source.token : refresh_token } + + - do: + security.get_token: + body: + grant_type: "refresh_token" + refresh_token: "${refresh_token}" + + - match: { type: "Bearer" } + - is_true: access_token + - set: { access_token: token } + - is_true: refresh_token + - set: { refresh_token: refresh_token } + - match: { expires_in: 3600 } + - is_false: scope + + - do: + headers: + Authorization: Bearer ${token} + security.authenticate: {} + + - match: { username: "token_user" } + - match: { roles.0: "superuser" } + - match: { full_name: "Token User" } + + - do: + headers: + Authorization: Bearer ${token} + search: + rest_total_hits_as_int: true + index: token_index + + - match: { hits.total: 8 } From 353a356551b27b14c42819876e38c7047c30a63a Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 10 Apr 2019 00:21:16 +0300 Subject: [PATCH 42/50] Merge fallout --- .../action/oidc/TransportOpenIdConnectLogoutActionTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java index ddf1742109915..e31ccc6733290 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java @@ -167,7 +167,7 @@ public void setup() throws Exception { when(securityIndex.isAvailable()).thenReturn(true); final ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); - tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, clusterService); + tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, securityIndex, clusterService); final TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); From a0f29df24a144ec3a34c80150074b981f02b4bac Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 10 Apr 2019 01:32:15 +0300 Subject: [PATCH 43/50] Stupid renaming --- .../org/elasticsearch/xpack/restart/FullClusterRestartIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 41c08d5919ba9..facd52c64dfbf 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -20,9 +20,9 @@ import org.elasticsearch.test.StreamsUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; -import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames; import org.elasticsearch.xpack.core.upgrade.UpgradeField; import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.watcher.actions.index.IndexAction; import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; @@ -106,7 +106,7 @@ public void testSecurityNativeRealm() throws Exception { if (settingsMap.containsKey("index")) { @SuppressWarnings("unchecked") int format = Integer.parseInt(String.valueOf(((Map)settingsMap.get("index")).get("format"))); - assertEquals("The security index needs to be upgraded", RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, format); + assertEquals("The security index needs to be upgraded", SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT, format); } } From f1c4e44a077649e935da281dc6649588f27eba2d Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 11 Apr 2019 16:22:56 +0300 Subject: [PATCH 44/50] Renames --- .../security/authc/ExpiredTokenRemover.java | 10 +++++----- .../xpack/security/authc/TokenService.java | 18 +++++++++--------- .../security/authc/TokenServiceTests.java | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index 8f06973f3423e..23e7bb2fe0fe5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -48,7 +48,7 @@ final class ExpiredTokenRemover extends AbstractRunnable { private final SecurityIndexManager securityTokensIndex; private final AtomicBoolean inProgress; private final TimeValue timeout; - private boolean mainIndexMightContainTokens; + private boolean checkMainIndexForExpiredTokens; ExpiredTokenRemover(Settings settings, Client client, SecurityIndexManager securityMainIndex, SecurityIndexManager securityTokensIndex) { @@ -57,7 +57,7 @@ final class ExpiredTokenRemover extends AbstractRunnable { this.securityTokensIndex = securityTokensIndex; this.inProgress = new AtomicBoolean(false); this.timeout = TokenService.DELETE_TIMEOUT.get(settings); - this.mainIndexMightContainTokens = true; + this.checkMainIndexForExpiredTokens = true; } @Override @@ -66,7 +66,7 @@ public void doRun() { if (securityTokensIndex.isAvailable()) { indicesWithTokens.add(securityTokensIndex.aliasName()); } - if (securityMainIndex.isAvailable() && mainIndexMightContainTokens) { + if (securityMainIndex.isAvailable() && checkMainIndexForExpiredTokens) { indicesWithTokens.add(securityMainIndex.aliasName()); } if (indicesWithTokens.isEmpty()) { @@ -89,10 +89,10 @@ public void doRun() { debugDbqResponse(bulkResponse); // tokens can still linger on the main index for their maximum lifetime after the tokens index has been created, because // only after the tokens index has been created all nodes will store tokens there and not on the main security index - if (mainIndexMightContainTokens && securityTokensIndex.indexExists() + if (checkMainIndexForExpiredTokens && securityTokensIndex.indexExists() && securityTokensIndex.getCreationTime().isBefore(now.minus(MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS)) && bulkResponse.getBulkFailures().isEmpty() && bulkResponse.getSearchFailures().isEmpty()) { - mainIndexMightContainTokens = false; + checkMainIndexForExpiredTokens = false; } markComplete(); }, this::onFailure)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 9a207bd0138e5..ad425840057d4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -239,7 +239,7 @@ public void createOAuth2Tokens(Authentication authentication, Authentication ori // the created token is compatible with the oldest node version in the cluster final Version tokenVersion = getTokenVersionCompatibility(); // tokens moved to a separate index in newer versions - final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(tokenVersion); + final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion); // the id of the created tokens ought be unguessable final String userTokenId = UUIDs.randomBase64UUID(); createOAuth2Tokens(userTokenId, tokenVersion, tokensIndex, authentication, originatingClientAuth, metadata, includeRefreshToken, @@ -342,7 +342,7 @@ public void getAuthenticationAndMetaData(String token, ActionListener listener) { - final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(tokenVersion); + final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion); if (tokensIndex.isAvailable() == false) { logger.warn("failed to get access token [{}] because index [{}] is not available", userTokenId, tokensIndex.aliasName()); listener.onResponse(null); @@ -558,7 +558,7 @@ public void invalidateActiveTokensForRealmAndUser(@Nullable String realmName, @N * Invalidates a collection of access_token and refresh_token that were retrieved by * {@link TokenService#invalidateActiveTokensForRealmAndUser} * - * @param userTokens The the user tokens for which access and refresh tokens should be invalidated + * @param userTokens The user tokens for which access and refresh tokens should be invalidated * @param listener the listener to notify upon completion */ private void invalidateAllTokens(Collection userTokens, ActionListener listener) { @@ -847,14 +847,14 @@ private void innerRefresh(String tokenDocId, Map source, long se // the superseding token document reference is formated as: "|" ; the alias points to a single index // containing the document with the said id updateMap.put("superseded_by", - getTokensIndexManagerForVersion(newTokenVersion).aliasName() + "|" + getTokenDocumentId(newUserTokenId)); + getTokensIndexForVersion(newTokenVersion).aliasName() + "|" + getTokenDocumentId(newUserTokenId)); } else { // preservers the format of the reference so that old nodes in a mixed cluster can still understand it updateMap.put("superseded_by", getTokenDocumentId(newUserTokenId)); } assert seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; assert primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM : "expected an assigned primary term"; - final SecurityIndexManager refreshedTokenIndex = getTokensIndexManagerForVersion(refreshTokenStatus.getVersion()); + final SecurityIndexManager refreshedTokenIndex = getTokensIndexForVersion(refreshTokenStatus.getVersion()); final UpdateRequestBuilder updateRequest = client .prepareUpdate(refreshedTokenIndex.aliasName(), SINGLE_MAPPING_NAME, tokenDocId) .setDoc("refresh_token", updateMap) @@ -869,7 +869,7 @@ private void innerRefresh(String tokenDocId, Map source, long se updateResponse.getGetResult().sourceAsMap())); final Tuple parsedTokens = parseTokensFromDocument(source, null); final UserToken toRefreshUserToken = parsedTokens.v1(); - createOAuth2Tokens(newUserTokenId, newTokenVersion, getTokensIndexManagerForVersion(newTokenVersion), + createOAuth2Tokens(newUserTokenId, newTokenVersion, getTokensIndexForVersion(newTokenVersion), toRefreshUserToken.getAuthentication(), clientAuth, toRefreshUserToken.getMetadata(), true, listener); } else if (backoff.hasNext()) { logger.info("failed to update the original token document [{}], the update result was [{}]. Retrying", @@ -1370,7 +1370,7 @@ private void ensureEnabled() { * removed automatically by the {@code ExpiredTokenRemover} periodic job. Therefore, in general, when searching for a token we need to * consider both the new and the old indices. */ - private SecurityIndexManager getTokensIndexManagerForVersion(Version version) { + private SecurityIndexManager getTokensIndexForVersion(Version version) { if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { return securityTokensIndex; } else { @@ -1386,7 +1386,7 @@ private void checkIfTokenIsValid(UserToken userToken, ActionListener listener.onFailure(traceLog("validate token", userToken.getId(), expiredTokenException())); return; } - final SecurityIndexManager tokensIndex = getTokensIndexManagerForVersion(userToken.getVersion()); + final SecurityIndexManager tokensIndex = getTokensIndexForVersion(userToken.getVersion()); if (tokensIndex.indexExists() == false) { // index doesn't exist so the token is considered invalid as we cannot verify its validity logger.warn("failed to validate access token because the index [" + tokensIndex.aliasName() + "] doesn't exist"); @@ -1465,7 +1465,7 @@ private String getFromHeader(ThreadContext threadContext) { /** * Serializes a token to a String containing the minimum compatible node version for decoding it back and either an encrypted - * representation of the token id for versions earlier to {@code #VERSION_ACCESS_TOKENS_UUIDS} the token ie itself for versions after + * representation of the token id for versions earlier to {@code #VERSION_ACCESS_TOKENS_UUIDS} or the token itself for versions after * {@code #VERSION_ACCESS_TOKENS_UUIDS} */ public String getAccessTokenAsString(UserToken userToken) throws IOException, GeneralSecurityException { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 4ac877007ff28..f37fa8295fb62 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -135,7 +135,7 @@ public void setupClient() { // tokens docs on a separate index), let's test the TokenService works in a mixed cluster with nodes with versions prior to these // developments if (randomBoolean()) { - oldNode = addAnotherDateNodeWithVersion(this.clusterService, randomFrom(Version.V_6_7_0, Version.V_7_0_0)); + oldNode = addAnotherDataNodeWithVersion(this.clusterService, randomFrom(Version.V_6_7_0, Version.V_7_0_0)); } } @@ -713,7 +713,7 @@ private SecurityIndexManager mockSecurityManager() { return mockSecurityIndex; } - private DiscoveryNode addAnotherDateNodeWithVersion(ClusterService clusterService, Version version) { + private DiscoveryNode addAnotherDataNodeWithVersion(ClusterService clusterService, Version version) { final ClusterState currentState = clusterService.state(); final DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(currentState.getNodes()); final DiscoveryNode anotherDataNode = new DiscoveryNode("another_data_node#" + version, buildNewFakeTransportAddress(), From 4e3e5939f196fd1df2b6ef37ce5c724a7886f443 Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Wed, 17 Apr 2019 17:09:34 +0300 Subject: [PATCH 45/50] Update x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java Co-Authored-By: albertzaharovits --- .../elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index cca3297b0ce40..5b5f6e0f6070b 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -30,7 +30,7 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase { private Collection twoClients = null; @Before - private void collateClientsByVersion() throws IOException { + private void collectClientsByVersion() throws IOException { Map clientsByVersion = getRestClientByVersion(); if (clientsByVersion.size() == 2) { // usual case, clients have different versions From 3391653126245a9da160617b479f28fa0007fd5e Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> Date: Wed, 17 Apr 2019 17:09:50 +0300 Subject: [PATCH 46/50] Update x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml Co-Authored-By: albertzaharovits --- .../rest-api-spec/test/mixed_cluster/50_token_auth.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml index 5e3fce8ef6fba..f426d9b2525b4 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/50_token_auth.yml @@ -108,7 +108,7 @@ - match: { hits.total: 8 } --- -"Get the indexed refresh token and use if to get another access token and authenticate": +"Get the indexed refresh token and use it to get another access token and authenticate": - skip: features: headers From 18376ef60fc4596d06be722b28ed2741e96b475d Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Wed, 17 Apr 2019 17:13:08 +0300 Subject: [PATCH 47/50] Update x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java Co-Authored-By: albertzaharovits --- .../elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index 5b5f6e0f6070b..7c4040ddf5c70 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -131,7 +131,7 @@ public void testAccessTokensWorkInMixedCluster() throws Exception { } public void testTokensStayInvalidatedInMixedCluster() throws Exception { - // Verify that an old token continues to work during all stages of the rolling upgrade + // Verify that an old, invalidated token remains invalidated during all stages of the rolling upgrade assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); Map source = retrieveStoredTokens(client(), 5); assertAccessTokenDoesNotWork((String) source.get("token")); From 33a44338044efb0cab5b685968ffc722d9676e3c Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 17 Apr 2019 17:50:29 +0300 Subject: [PATCH 48/50] Nits --- .../xpack/security/authc/TokenServiceTests.java | 4 ++++ .../upgrades/TokenBackwardsCompatibilityIT.java | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index f37fa8295fb62..494b8070c57d9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -597,8 +597,12 @@ public void testIndexNotAvailable() throws Exception { final SecurityIndexManager tokensIndex; if (oldNode != null) { tokensIndex = securityMainIndex; + when(securityTokensIndex.isAvailable()).thenReturn(false); + when(securityTokensIndex.indexExists()).thenReturn(false); } else { tokensIndex = securityTokensIndex; + when(securityMainIndex.isAvailable()).thenReturn(false); + when(securityMainIndex.indexExists()).thenReturn(false); } try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) { PlainActionFuture future = new PlainActionFuture<>(); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index 7c4040ddf5c70..69c515d80a3d2 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -78,7 +78,7 @@ public void testGeneratingTokensInOldCluster() throws Exception { public void testRefreshingTokensInOldCluster() throws Exception { assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD); - // Creates another access and refresh tokens and tries to use the refresh tokens several times + // Creates access and refresh tokens and uses the refresh token. The new resulting tokens are used in different phases Map responseMap = createTokens(client(), "test_user", "x-pack-test-password"); String accessToken = (String) responseMap.get("access_token"); assertNotNull(accessToken); @@ -88,7 +88,8 @@ public void testRefreshingTokensInOldCluster() throws Exception { storeTokens(client(), 3, accessToken, refreshToken); - // refresh the new token + // refresh the token just created. The old token is invalid (tested further) and the new refresh token is tested in the upgraded + // cluster Map refreshResponseMap = refreshToken(client(), refreshToken); String refreshedAccessToken = (String) refreshResponseMap.get("access_token"); String refreshedRefreshToken = (String) refreshResponseMap.get("refresh_token"); @@ -103,7 +104,7 @@ public void testRefreshingTokensInOldCluster() throws Exception { public void testInvalidatingTokensInOldCluster() throws Exception { assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD); - // Creates another access and refresh tokens and tries to use the refresh tokens several times + // Creates access and refresh tokens and tries to use the access tokens several times Map responseMap = createTokens(client(), "test_user", "x-pack-test-password"); String accessToken = (String) responseMap.get("access_token"); assertNotNull(accessToken); @@ -165,7 +166,7 @@ public void testGeneratingTokensInMixedCluster() throws Exception { } public void testRefreshingTokensInMixedCluster() throws Exception { - // verify new nodes can refresh tokens from the old cluster + // verify new nodes can refresh tokens created by old nodes and vice versa assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); for (RestClient client1 : twoClients) { Map responseMap = createTokens(client1, "test_user", "x-pack-test-password"); From 01d0129f9c0efabae9c61b7d1df53d40fa3b78a7 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 30 Apr 2019 08:58:06 +0300 Subject: [PATCH 49/50] Clarify the superseded_by format --- .../xpack/security/authc/TokenService.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 7aa88a1e32ea5..10236e1719b5c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -844,12 +844,12 @@ private void innerRefresh(String tokenDocId, Map source, long se updateMap.put("refreshed", true); updateMap.put("refresh_time", clock.instant().toEpochMilli()); if (newTokenVersion.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { - // the superseding token document reference is formated as: "|" ; the alias points to a single index - // containing the document with the said id - updateMap.put("superseded_by", - getTokensIndexForVersion(newTokenVersion).aliasName() + "|" + getTokenDocumentId(newUserTokenId)); + // the superseding token document reference is formated as "|"; + // for now, only the ".security-tokens|" is a valid reference format + updateMap.put("superseded_by", securityTokensIndex.aliasName() + "|" + getTokenDocumentId(newUserTokenId)); } else { - // preservers the format of the reference so that old nodes in a mixed cluster can still understand it + // preservers the format of the reference (without the alias prefix) + // so that old nodes in a mixed cluster can still understand it updateMap.put("superseded_by", getTokenDocumentId(newUserTokenId)); } assert seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; From 46ffb007a453b53f5f7e56b78fd410796d81b257 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 30 Apr 2019 09:01:23 +0300 Subject: [PATCH 50/50] dynamic: false --- .../src/main/resources/security-tokens-index-template-7.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json index cef76c8975221..e7450d0be9c28 100644 --- a/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json +++ b/x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json @@ -75,7 +75,7 @@ }, "metadata" : { "type" : "object", - "dynamic" : true + "dynamic" : false }, "authentication" : { "type" : "binary"