A new search API for API keys - core search function (#75335)

This PR adds a new API for searching API keys. The API supports searching API
keys with a controlled list of field names and a subset of Query DSL. It also
provides a translation layer between the field names used in the REST layer and
those in the index layer. This is to prevent tight coupling between the user
facing request and index mappings so that they can evolve separately.

Compared to the Get API key API, this new search API automatically applies
calling user's security context similar to regular searches, e.g. if the user
has only manage_own_api_key privilege, only keys owned by the user are returned
in the search response.

Relates: #71023

Author: ywangd

server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java

@@ -107,6 +107,7 @@ public class SearchExecutionContext extends QueryRewriteContext {
     private NestedScope nestedScope;
     private final ValuesSourceRegistry valuesSourceRegistry;
     private final Map<String, MappedFieldType> runtimeMappings;
+    private Predicate<String> allowedFields;
 
     /**
      * Build a {@linkplain SearchExecutionContext}.
@@ -154,7 +155,8 @@ public SearchExecutionContext(
             ),
             allowExpensiveQueries,
             valuesSourceRegistry,
-            parseRuntimeMappings(runtimeMappings, mapperService)
+            parseRuntimeMappings(runtimeMappings, mapperService),
+            null
         );
     }
 
@@ -177,7 +179,8 @@ public SearchExecutionContext(SearchExecutionContext source) {
             source.fullyQualifiedIndex,
             source.allowExpensiveQueries,
             source.valuesSourceRegistry,
-            source.runtimeMappings
+            source.runtimeMappings,
+            source.allowedFields
         );
     }
 
@@ -199,7 +202,8 @@ private SearchExecutionContext(int shardId,
                                    Index fullyQualifiedIndex,
                                    BooleanSupplier allowExpensiveQueries,
                                    ValuesSourceRegistry valuesSourceRegistry,
-                                   Map<String, MappedFieldType> runtimeMappings) {
+                                   Map<String, MappedFieldType> runtimeMappings,
+                                   Predicate<String> allowedFields) {
         super(xContentRegistry, namedWriteableRegistry, client, nowInMillis);
         this.shardId = shardId;
         this.shardRequestIndex = shardRequestIndex;
@@ -218,6 +222,7 @@ private SearchExecutionContext(int shardId,
         this.allowExpensiveQueries = allowExpensiveQueries;
         this.valuesSourceRegistry = valuesSourceRegistry;
         this.runtimeMappings = runtimeMappings;
+        this.allowedFields = allowedFields;
     }
 
     private void reset() {
@@ -352,6 +357,10 @@ public boolean isFieldMapped(String name) {
     }
 
     private MappedFieldType fieldType(String name) {
+        // If the field is not allowed, behave as if it is not mapped
+        if (allowedFields != null && false == allowedFields.test(name)) {
+            return null;
+        }
         MappedFieldType fieldType = runtimeMappings.get(name);
         return fieldType == null ? mappingLookup.getFieldType(name) : fieldType;
     }
@@ -419,6 +428,10 @@ public void setMapUnmappedFieldAsString(boolean mapUnmappedFieldAsString) {
         this.mapUnmappedFieldAsString = mapUnmappedFieldAsString;
     }
 
+    public void setAllowedFields(Predicate<String> allowedFields) {
+        this.allowedFields = allowedFields;
+    }
+
     MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMapping) {
         if (fieldMapping != null || allowUnmappedFields) {
             return fieldMapping;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyAction.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.apikey;
+
+import org.elasticsearch.action.ActionType;
+
+public final class QueryApiKeyAction extends ActionType<QueryApiKeyResponse> {
+
+    public static final String NAME = "cluster:admin/xpack/security/api_key/query";
+    public static final QueryApiKeyAction INSTANCE = new QueryApiKeyAction();
+
+    private QueryApiKeyAction() {
+        super(NAME, QueryApiKeyResponse::new);
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyRequest.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.apikey;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.query.QueryBuilder;
+
+import java.io.IOException;
+
+public final class QueryApiKeyRequest extends ActionRequest {
+
+    @Nullable
+    private final QueryBuilder queryBuilder;
+    private boolean filterForCurrentUser;
+
+    public QueryApiKeyRequest() {
+        this((QueryBuilder) null);
+    }
+
+    public QueryApiKeyRequest(QueryBuilder queryBuilder) {
+        this.queryBuilder = queryBuilder;
+    }
+
+    public QueryApiKeyRequest(StreamInput in) throws IOException {
+        super(in);
+        queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
+    }
+
+    public QueryBuilder getQueryBuilder() {
+        return queryBuilder;
+    }
+
+    public boolean isFilterForCurrentUser() {
+        return filterForCurrentUser;
+    }
+
+    public void setFilterForCurrentUser() {
+        filterForCurrentUser = true;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalNamedWriteable(queryBuilder);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponse.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.apikey;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.security.action.ApiKey;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+
+/**
+ * Response for search API keys.<br>
+ * The result contains information about the API keys that were found.
+ */
+public final class QueryApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable {
+
+    private final ApiKey[] foundApiKeysInfo;
+
+    public QueryApiKeyResponse(StreamInput in) throws IOException {
+        super(in);
+        this.foundApiKeysInfo = in.readArray(ApiKey::new, ApiKey[]::new);
+    }
+
+    public QueryApiKeyResponse(Collection<ApiKey> foundApiKeysInfo) {
+        Objects.requireNonNull(foundApiKeysInfo, "found_api_keys_info must be provided");
+        this.foundApiKeysInfo = foundApiKeysInfo.toArray(new ApiKey[0]);
+    }
+
+    public static QueryApiKeyResponse emptyResponse() {
+        return new QueryApiKeyResponse(Collections.emptyList());
+    }
+
+    public ApiKey[] getApiKeyInfos() {
+        return foundApiKeysInfo;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject()
+            .array("api_keys", (Object[]) foundApiKeysInfo);
+        return builder.endObject();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeArray(foundApiKeysInfo);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        QueryApiKeyResponse that = (QueryApiKeyResponse) o;
+        return Arrays.equals(foundApiKeysInfo, that.foundApiKeysInfo);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(foundApiKeysInfo);
+    }
+
+    @Override
+    public String toString() {
+        return "QueryApiKeyResponse [foundApiKeysInfo=" + foundApiKeysInfo + "]";
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java

@@ -216,7 +216,6 @@ private Builder(RoleDescriptor rd, @Nullable FieldPermissionsCache fieldPermissi
 
         public Builder cluster(Set<String> privilegeNames, Iterable<ConfigurableClusterPrivilege> configurableClusterPrivileges) {
             ClusterPermission.Builder builder = ClusterPermission.builder();
-            List<ClusterPermission> clusterPermissions = new ArrayList<>();
             if (privilegeNames.isEmpty() == false) {
                 for (String name : privilegeNames) {
                     builder = ClusterPrivilegeResolver.resolve(name).buildPermission(builder);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission;
@@ -75,6 +76,9 @@ protected boolean extendedCheck(String action, TransportRequest request, Authent
                         invalidateApiKeyRequest.getUserName(), invalidateApiKeyRequest.getRealmName(),
                         invalidateApiKeyRequest.ownedByAuthenticatedUser()));
                 }
+            } else if (request instanceof QueryApiKeyRequest) {
+                final QueryApiKeyRequest queryApiKeyRequest = (QueryApiKeyRequest) request;
+                return queryApiKeyRequest.isFilterForCurrentUser();
             }
             throw new IllegalArgumentException(
                 "manage own api key privilege only supports API key requests (not " + request.getClass().getName() + ")");

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyRequestTests.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.apikey;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class QueryApiKeyRequestTests extends ESTestCase {
+
+    @Override
+    protected NamedWriteableRegistry writableRegistry() {
+        final SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of());
+        return new NamedWriteableRegistry(searchModule.getNamedWriteables());
+    }
+
+    public void testReadWrite() throws IOException {
+        final QueryApiKeyRequest request1 = new QueryApiKeyRequest();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            request1.writeTo(out);
+            try (StreamInput in = new InputStreamStreamInput(new ByteArrayInputStream(out.bytes().array()))) {
+                assertThat(new QueryApiKeyRequest(in).getQueryBuilder(), nullValue());
+            }
+        }
+
+        final BoolQueryBuilder boolQueryBuilder2 = QueryBuilders.boolQuery()
+            .filter(QueryBuilders.termQuery("foo", "bar"))
+            .should(QueryBuilders.idsQuery().addIds("id1", "id2"))
+            .must(QueryBuilders.wildcardQuery("a.b", "t*y"))
+            .mustNot(QueryBuilders.prefixQuery("value", "prod"));
+        final QueryApiKeyRequest request2 = new QueryApiKeyRequest(boolQueryBuilder2);
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            request2.writeTo(out);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), writableRegistry())) {
+                final QueryApiKeyRequest deserialized = new QueryApiKeyRequest(in);
+                assertThat(deserialized.getQueryBuilder().getClass(), is(BoolQueryBuilder.class));
+                assertThat((BoolQueryBuilder) deserialized.getQueryBuilder(), equalTo(boolQueryBuilder2));
+            }
+        }
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.apikey;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.security.action.ApiKey;
+import org.elasticsearch.xpack.core.security.action.ApiKeyTests;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class QueryApiKeyResponseTests extends AbstractWireSerializingTestCase<QueryApiKeyResponse> {
+
+    @Override
+    protected Writeable.Reader<QueryApiKeyResponse> instanceReader() {
+        return QueryApiKeyResponse::new;
+    }
+
+    @Override
+    protected QueryApiKeyResponse createTestInstance() {
+        final List<ApiKey> apiKeys = randomList(0, 3, this::randomApiKeyInfo);
+        return new QueryApiKeyResponse(apiKeys);
+    }
+
+    @Override
+    protected QueryApiKeyResponse mutateInstance(QueryApiKeyResponse instance) throws IOException {
+        final ArrayList<ApiKey> apiKeyInfos =
+            Arrays.stream(instance.getApiKeyInfos()).collect(Collectors.toCollection(ArrayList::new));
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                apiKeyInfos.add(randomApiKeyInfo());
+                return new QueryApiKeyResponse(apiKeyInfos);
+            case 1:
+                if (false == apiKeyInfos.isEmpty()) {
+                    return new QueryApiKeyResponse(apiKeyInfos.subList(1, apiKeyInfos.size()));
+                } else {
+                    apiKeyInfos.add(randomApiKeyInfo());
+                    return new QueryApiKeyResponse(apiKeyInfos);
+                }
+            default:
+                if (false == apiKeyInfos.isEmpty()) {
+                    final int index = randomIntBetween(0, apiKeyInfos.size() - 1);
+                    apiKeyInfos.set(index, randomApiKeyInfo());
+                } else {
+                    apiKeyInfos.add(randomApiKeyInfo());
+                }
+                return new QueryApiKeyResponse(apiKeyInfos);
+        }
+    }
+
+    private ApiKey randomApiKeyInfo() {
+        final String name = randomAlphaOfLengthBetween(3, 8);
+        final String id = randomAlphaOfLength(22);
+        final String username = randomAlphaOfLengthBetween(3, 8);
+        final String realm_name = randomAlphaOfLengthBetween(3, 8);
+        final Instant creation = Instant.ofEpochMilli(randomMillisUpToYear9999());
+        final Instant expiration = randomBoolean() ? Instant.ofEpochMilli(randomMillisUpToYear9999()) : null;
+        final Map<String, Object> metadata = ApiKeyTests.randomMetadata();
+        return new ApiKey(name, id, creation, expiration, false, username, realm_name, metadata);
+    }
+}