diff --git a/entity-service-impl/build.gradle.kts b/entity-service-impl/build.gradle.kts index 7f1e321c..a60eeaf4 100644 --- a/entity-service-impl/build.gradle.kts +++ b/entity-service-impl/build.gradle.kts @@ -7,7 +7,7 @@ plugins { dependencies { api(project(":entity-service-api")) api("org.hypertrace.core.serviceframework:service-framework-spi:0.1.19") - implementation("org.hypertrace.core.documentstore:document-store:0.5.1") + implementation("org.hypertrace.core.documentstore:document-store:0.5.2") implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.3.1") implementation(project(":entity-type-service-rx-client")) diff --git a/entity-service-impl/src/main/java/org/hypertrace/entity/service/util/DocStoreConverter.java b/entity-service-impl/src/main/java/org/hypertrace/entity/service/util/DocStoreConverter.java index a9447ba4..5aacde9e 100644 --- a/entity-service-impl/src/main/java/org/hypertrace/entity/service/util/DocStoreConverter.java +++ b/entity-service-impl/src/main/java/org/hypertrace/entity/service/util/DocStoreConverter.java @@ -128,8 +128,12 @@ private static Filter transform(AttributeFilter filter) { if (filter.hasAttributeValue()) { if (ATTRIBUTES_LABELS_FIELD_NAME.equals(filter.getName()) && filter.getOperator() == Operator.EQ) { return transformToEqFilterWithValueListRhs(filter); + } else if (ATTRIBUTES_LABELS_FIELD_NAME.equals(filter.getName()) && filter.getOperator() == Operator.NEQ) { + return transformToNeqFilterWithValueListRhs(filter); } else if (ATTRIBUTES_LABELS_FIELD_NAME.equals(filter.getName()) && filter.getOperator() == Operator.IN) { return transformToOrFilterChainForStrArray(filter); + } else if (ATTRIBUTES_LABELS_FIELD_NAME.equals(filter.getName()) && filter.getOperator() == Operator.NOT_IN) { + return transformToAndFilterChainForStrArray(filter); } else { return transformNonListRhsFilterTypes(filter); } @@ -183,22 +187,65 @@ private static Filter transformToOrFilterChainForStrArray(AttributeFilter attrib return f; } + private static Filter transformToAndFilterChainForStrArray(AttributeFilter attributeFilter) { + String fieldName = attributeFilter.getName() + VALUE_LIST_VALUES_CONST; + + Filter f = new Filter(); + f.setFieldName(""); + f.setOp(Op.AND); + + List filters = attributeFilter.getAttributeValue().getValueList().getValuesList().stream() + .map(rhsAttributeValue -> createNeqFilterForAttributeValue(fieldName, rhsAttributeValue)) + .collect(Collectors.toList()); + + f.setChildFilters(filters.toArray(new Filter[]{})); + + return f; + } + private static Filter transformToEqFilterWithValueListRhs(AttributeFilter attributeFilter) { String fieldName = attributeFilter.getName() + VALUE_LIST_VALUES_CONST; return createEqFilterForAttributeValue(fieldName, attributeFilter.getAttributeValue()); } + private static Filter transformToNeqFilterWithValueListRhs(AttributeFilter attributeFilter) { + String fieldName = attributeFilter.getName() + VALUE_LIST_VALUES_CONST; + Filter f = new Filter(); + f.setFieldName(fieldName); + f.setOp(Op.NEQ); + f.setValue(prepareRhsValueForSpecialValueListCase(attributeFilter.getAttributeValue())); + // Set child filters to empty array + f.setChildFilters(new Filter[]{}); + return f; + } + private static Filter createEqFilterForAttributeValue(String fieldName, AttributeValue attributeValue) { Filter f = new Filter(); f.setFieldName(fieldName); f.setOp(Op.EQ); + f.setValue(prepareRhsValueForSpecialValueListCase(attributeValue)); + // Set child filters to empty array + f.setChildFilters(new Filter[]{}); + return f; + } + private static Filter createNeqFilterForAttributeValue(String fieldName, AttributeValue attributeValue) { + Filter f = new Filter(); + f.setFieldName(fieldName); + f.setOp(Op.NEQ); + f.setValue(prepareRhsValueForSpecialValueListCase(attributeValue)); + // Set child filters to empty array + f.setChildFilters(new Filter[]{}); + return f; + } + + private static Object prepareRhsValueForSpecialValueListCase(AttributeValue attributeValue) { org.hypertrace.entity.data.service.v1.AttributeValue.TypeCase typeCase = attributeValue.getTypeCase(); if (typeCase == TypeCase.VALUE) { try { JsonNode mapNode = OBJECT_MAPPER.readTree(JSONFORMAT_PRINTER.print(attributeValue)); Map map = OBJECT_MAPPER.convertValue(mapNode, Map.class); - f.setValue(map); + return map; } catch (JsonProcessingException | InvalidProtocolBufferException e) { throw new RuntimeException(e); } @@ -206,10 +253,6 @@ private static Filter createEqFilterForAttributeValue(String fieldName, Attribut throw new UnsupportedOperationException( String.format("The RHS of filter for string array types can only be VALUE: %s", attributeValue)); } - - // Set child filters to empty array - f.setChildFilters(new Filter[]{}); - return f; } private static void transform(AttributeValue attributeValue, Filter filter, diff --git a/entity-service-impl/src/test/java/org/hypertrace/entity/service/util/DocStoreConverterTest.java b/entity-service-impl/src/test/java/org/hypertrace/entity/service/util/DocStoreConverterTest.java index 0e8a23b1..e1db48ae 100644 --- a/entity-service-impl/src/test/java/org/hypertrace/entity/service/util/DocStoreConverterTest.java +++ b/entity-service-impl/src/test/java/org/hypertrace/entity/service/util/DocStoreConverterTest.java @@ -479,6 +479,57 @@ public void testStringArrayValueTypeColumnEqAndChain() throws JsonProcessingExce transformedFilter.getChildFilters()[2].getChildFilters()[1].getValue()); } + @Test + public void testStringArrayValueTypeColumnNeqAndChain() throws JsonProcessingException { + Query query = Query.newBuilder() + .addEntityId("some id") + .setFilter( + AttributeFilter.newBuilder().setOperator(Operator.AND) + .addChildFilter( + AttributeFilter.newBuilder() + .setName(ATTRIBUTES_LABELS_FIELD_NAME) + .setOperator(Operator.NEQ) + .setAttributeValue(AttributeValue.newBuilder() + .setValue(Value.newBuilder().setString("l1")) + ) + ) + .addChildFilter( + AttributeFilter.newBuilder() + .setName(ATTRIBUTES_LABELS_FIELD_NAME) + .setOperator(Operator.NEQ) + .setAttributeValue(AttributeValue.newBuilder() + .setValue(Value.newBuilder().setString("l2")) + ) + ) + ) + .build(); + org.hypertrace.core.documentstore.Query transformedQuery = + DocStoreConverter.transform(TENANT_ID, query, Collections.emptyList()); + + Filter transformedFilter = transformedQuery.getFilter(); + Assertions.assertEquals(Filter.Op.AND, transformedFilter.getOp()); + + Assertions.assertEquals(3, transformedFilter.getChildFilters().length); + Assertions.assertEquals(EntityServiceConstants.ENTITY_ID, + transformedFilter.getChildFilters()[1].getFieldName()); + Assertions.assertEquals(Collections.singletonList("some id"), + transformedFilter.getChildFilters()[1].getValue()); + + Assertions.assertEquals(Op.AND, transformedFilter.getChildFilters()[2].getOp()); + + Assertions.assertEquals(ATTRIBUTES_LABELS_FIELD_NAME + ".valueList.values", + transformedFilter.getChildFilters()[2].getChildFilters()[0].getFieldName()); + Assertions.assertEquals(Op.NEQ, transformedFilter.getChildFilters()[2].getChildFilters()[0].getOp()); + Assertions.assertEquals(OBJECT_MAPPER.convertValue(OBJECT_MAPPER.readTree("{\"value\": {\"string\":\"l1\"}}"), Map.class), + transformedFilter.getChildFilters()[2].getChildFilters()[0].getValue()); + + Assertions.assertEquals(ATTRIBUTES_LABELS_FIELD_NAME + ".valueList.values", + transformedFilter.getChildFilters()[2].getChildFilters()[1].getFieldName()); + Assertions.assertEquals(Op.NEQ, transformedFilter.getChildFilters()[2].getChildFilters()[1].getOp()); + Assertions.assertEquals(OBJECT_MAPPER.convertValue(OBJECT_MAPPER.readTree("{\"value\": {\"string\":\"l2\"}}"), Map.class), + transformedFilter.getChildFilters()[2].getChildFilters()[1].getValue()); + } + @Test public void testStringArrayValueTypeColumnOrChain() throws JsonProcessingException { Query query = Query.newBuilder() @@ -554,6 +605,81 @@ public void testStringArrayValueTypeColumnOrChain() throws JsonProcessingExcepti transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[2].getValue()); } + @Test + public void testStringArrayValueTypeColumnAndChain() throws JsonProcessingException { + Query query = Query.newBuilder() + .addEntityId("some id") + .setFilter( + AttributeFilter.newBuilder().setOperator(Operator.AND) + .addChildFilter( + AttributeFilter.newBuilder() + .setName("attributes.some_col") + .setOperator(Operator.EQ) + .setAttributeValue(AttributeValue.newBuilder() + .setValue(Value.newBuilder().setString("some_val")) + ) + ) + .addChildFilter( + AttributeFilter.newBuilder() + .setName(ATTRIBUTES_LABELS_FIELD_NAME) + .setOperator(Operator.NOT_IN) + .setAttributeValue(AttributeValue.newBuilder() + .setValueList( + AttributeValueList.newBuilder() + .addValues( + AttributeValue.newBuilder().setValue(Value.newBuilder().setString("l1")) + ) + .addValues( + AttributeValue.newBuilder().setValue(Value.newBuilder().setString("l2")) + ) + .addValues( + AttributeValue.newBuilder().setValue(Value.newBuilder().setString("l3")) + ) + ) + ) + ) + ) + .build(); + org.hypertrace.core.documentstore.Query transformedQuery = + DocStoreConverter.transform(TENANT_ID, query, Collections.emptyList()); + + Filter transformedFilter = transformedQuery.getFilter(); + Assertions.assertEquals(Filter.Op.AND, transformedFilter.getOp()); + + Assertions.assertEquals(3, transformedFilter.getChildFilters().length); + Assertions.assertEquals(EntityServiceConstants.ENTITY_ID, + transformedFilter.getChildFilters()[1].getFieldName()); + Assertions.assertEquals(Collections.singletonList("some id"), + transformedFilter.getChildFilters()[1].getValue()); + + Assertions.assertEquals(Op.AND, transformedFilter.getChildFilters()[2].getOp()); + + Assertions.assertEquals("attributes.some_col.value.string", + transformedFilter.getChildFilters()[2].getChildFilters()[0].getFieldName()); + Assertions.assertEquals(Op.EQ, transformedFilter.getChildFilters()[2].getChildFilters()[0].getOp()); + Assertions.assertEquals("some_val", transformedFilter.getChildFilters()[2].getChildFilters()[0].getValue()); + + Assertions.assertEquals(Op.AND, transformedFilter.getChildFilters()[2].getChildFilters()[1].getOp()); + + Assertions.assertEquals(ATTRIBUTES_LABELS_FIELD_NAME + ".valueList.values", + transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[0].getFieldName()); + Assertions.assertEquals(Op.NEQ, transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[0].getOp()); + Assertions.assertEquals(OBJECT_MAPPER.convertValue(OBJECT_MAPPER.readTree("{\"value\": {\"string\":\"l1\"}}"), Map.class), + transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[0].getValue()); + + Assertions.assertEquals(ATTRIBUTES_LABELS_FIELD_NAME + ".valueList.values", + transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[1].getFieldName()); + Assertions.assertEquals(Op.NEQ, transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[1].getOp()); + Assertions.assertEquals(OBJECT_MAPPER.convertValue(OBJECT_MAPPER.readTree("{\"value\": {\"string\":\"l2\"}}"), Map.class), + transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[1].getValue()); + + Assertions.assertEquals(ATTRIBUTES_LABELS_FIELD_NAME + ".valueList.values", + transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[2].getFieldName()); + Assertions.assertEquals(Op.NEQ, transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[2].getOp()); + Assertions.assertEquals(OBJECT_MAPPER.convertValue(OBJECT_MAPPER.readTree("{\"value\": {\"string\":\"l3\"}}"), Map.class), + transformedFilter.getChildFilters()[2].getChildFilters()[1].getChildFilters()[2].getValue()); + } + @Test public void testNeqFilterConversionForValueListThrowsException() { Assertions.assertThrows(UnsupportedOperationException.class, () -> { diff --git a/entity-service/build.gradle.kts b/entity-service/build.gradle.kts index 5857bb09..c4fb7896 100644 --- a/entity-service/build.gradle.kts +++ b/entity-service/build.gradle.kts @@ -59,7 +59,7 @@ dependencies { implementation("org.hypertrace.core.grpcutils:grpc-server-utils:0.3.1") implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.3.1") implementation("org.hypertrace.core.serviceframework:platform-service-framework:0.1.19") - implementation("org.hypertrace.core.documentstore:document-store:0.5.1") + implementation("org.hypertrace.core.documentstore:document-store:0.5.2") runtimeOnly("io.grpc:grpc-netty:1.33.1") runtimeOnly("io.netty:netty-codec-http2:4.1.59.Final") diff --git a/entity-service/src/integrationTest/java/org/hypertrace/entity/service/service/EntityDataServiceTest.java b/entity-service/src/integrationTest/java/org/hypertrace/entity/service/service/EntityDataServiceTest.java index 77846794..90115dc7 100644 --- a/entity-service/src/integrationTest/java/org/hypertrace/entity/service/service/EntityDataServiceTest.java +++ b/entity-service/src/integrationTest/java/org/hypertrace/entity/service/service/EntityDataServiceTest.java @@ -54,10 +54,10 @@ */ public class EntityDataServiceTest { - private static EntityDataServiceClient entityDataServiceClient; private static final String TENANT_ID = "__testTenant__" + EntityDataServiceTest.class.getSimpleName(); private static final String TEST_ENTITY_TYPE_V2 = "TEST_ENTITY"; + private static EntityDataServiceClient entityDataServiceClient; @BeforeAll public static void setUp() { @@ -574,7 +574,7 @@ public void testEntityQueryAttributeWithExistsFiltering() { Query.newBuilder().setFilter(notExistsFilter).build()); assertTrue(entities.size() > 0); - + // test with AND operator AttributeFilter eqFilter = AttributeFilter.newBuilder() .setName(EntityConstants.attributeMapPathFor("test" + "-" + stringRandomizer2)) @@ -653,7 +653,8 @@ public void whenNNewEntitiesAreUpserted_thenExpectNNewEntities() { } // Try getAndBulkUpsert, verify that the returned entities were in previous state. - Iterator iterator = entityDataServiceClient.getAndBulkUpsert(TENANT_ID, externalIdToEntity.values()); + Iterator iterator = entityDataServiceClient + .getAndBulkUpsert(TENANT_ID, externalIdToEntity.values()); while (iterator.hasNext()) { Entity entity = iterator.next(); assertNotNull(entityMap.get(entity.getEntityId())); @@ -874,6 +875,167 @@ public void testEntityQueryOrderBy() { } + @Test + public void testDifferentFilterForMatchingLabelsQuery() { + Entity entity = createEntityWithAttributeLabels("Some Service 1", + List.of("v1", "v2", "v3")); + Entity createdEntity1 = entityDataServiceClient.upsert(entity); + assertNotNull(createdEntity1); + assertNotNull(createdEntity1.getEntityId().trim()); + + entity = createEntityWithAttributeLabels("Some Service 2", + List.of("v01", "v02", "v03")); + Entity createdEntity2 = entityDataServiceClient.upsert(entity); + assertNotNull(createdEntity2); + assertNotNull(createdEntity2.getEntityId().trim()); + + entity = createEntityWithAttributeLabels("Some Service 3", + List.of("v01", "v2", "v4")); + Entity createdEntity3 = entityDataServiceClient.upsert(entity); + assertNotNull(createdEntity3); + assertNotNull(createdEntity3.getEntityId().trim()); + + entity = createEntityWithAttributeLabels("Some Service 4", + List.of("b1", "b2", "b3")); + Entity createdEntity4 = entityDataServiceClient.upsert(entity); + assertNotNull(createdEntity4); + assertNotNull(createdEntity4.getEntityId().trim()); + + entity = createEntityWithOutAttributeLabels("Some Service 5", + "foo", "bar"); + Entity createdEntity5 = entityDataServiceClient.upsert(entity); + assertNotNull(createdEntity5); + assertNotNull(createdEntity5.getEntityId().trim()); + + // test NOT_IN + AttributeFilter attributeFilter = AttributeFilter.newBuilder() + .setName("attributes.labels") + .setOperator(Operator.NOT_IN) + .setAttributeValue(AttributeValue.newBuilder() + .setValueList( + AttributeValueList.newBuilder() + .addValues( + AttributeValue.newBuilder().setValue(Value.newBuilder().setString("b1")) + ) + .addValues( + AttributeValue.newBuilder().setValue(Value.newBuilder().setString("v4")) + ) + ) + ).build(); + + Query query = Query.newBuilder() + .setEntityType(EntityType.K8S_POD.name()) + .setFilter(attributeFilter).build(); + List entitiesList = entityDataServiceClient.query(TENANT_ID, query); + assertTrue(entitiesList.size() == 3); + assertTrue(entitiesList.contains(createdEntity1) + && entitiesList.contains(createdEntity2) + && entitiesList.contains(createdEntity5)); + + // test IN + attributeFilter = AttributeFilter.newBuilder() + .setName("attributes.labels") + .setOperator(Operator.IN) + .setAttributeValue(AttributeValue.newBuilder() + .setValueList( + AttributeValueList.newBuilder() + .addValues( + AttributeValue.newBuilder().setValue(Value.newBuilder().setString("b1")) + ) + .addValues( + AttributeValue.newBuilder().setValue(Value.newBuilder().setString("v4")) + ) + ) + ).build(); + + query = Query.newBuilder().setEntityType(EntityType.K8S_POD.name()).setFilter(attributeFilter) + .build(); + entitiesList = entityDataServiceClient.query(TENANT_ID, query); + assertTrue(entitiesList.size() == 2); + assertTrue(entitiesList.contains(createdEntity3) && entitiesList.contains(createdEntity4)); + + // test NEQ + attributeFilter = AttributeFilter.newBuilder().setOperator(Operator.AND) + .addChildFilter( + AttributeFilter.newBuilder() + .setName("attributes.labels") + .setOperator(Operator.NEQ) + .setAttributeValue(AttributeValue.newBuilder() + .setValue(Value.newBuilder().setString("v2")) + ) + ) + .addChildFilter( + AttributeFilter.newBuilder() + .setName("attributes.labels") + .setOperator(Operator.NEQ) + .setAttributeValue(AttributeValue.newBuilder() + .setValue(Value.newBuilder().setString("b1")) + ) + ).build(); + + query = Query.newBuilder().setEntityType(EntityType.K8S_POD.name()).setFilter(attributeFilter) + .build(); + entitiesList = entityDataServiceClient.query(TENANT_ID, query); + assertTrue(entitiesList.size() == 2); + assertTrue(entitiesList.contains(createdEntity2) && entitiesList.contains(createdEntity5)); + + + // test EQ + attributeFilter = AttributeFilter.newBuilder() + .setName("attributes.labels") + .setOperator(Operator.EQ) + .setAttributeValue(AttributeValue.newBuilder() + .setValue(Value.newBuilder().setString("v2"))) + .build(); + + query = Query.newBuilder().setEntityType(EntityType.K8S_POD.name()).setFilter(attributeFilter) + .build(); + entitiesList = entityDataServiceClient.query(TENANT_ID, query); + assertTrue(entitiesList.size() == 2); + assertTrue(entitiesList.contains(createdEntity1) + && entitiesList.contains(createdEntity3)); + + } + + private Entity createEntityWithAttributeLabels(String entityName, List labels) { + AttributeValueList.Builder listBuilder = AttributeValueList.newBuilder(); + labels.forEach(label -> listBuilder.addValues( + AttributeValue.newBuilder().setValue(Value.newBuilder().setString(label))) + ); + AttributeValue value = AttributeValue.newBuilder().setValueList(listBuilder.build()).build(); + + Entity entity = Entity.newBuilder() + .setTenantId(TENANT_ID) + .setEntityType(EntityType.K8S_POD.name()) + .setEntityName(entityName) + .putIdentifyingAttributes( + EntityConstants.getValue(CommonAttribute.COMMON_ATTRIBUTE_EXTERNAL_ID), + generateRandomUUIDAttrValue()) + .putAttributes( + "labels", value) + .build(); + + return entity; + } + + private Entity createEntityWithOutAttributeLabels(String entityName, String attrName, + String attrValue) { + Entity entity = Entity.newBuilder() + .setTenantId(TENANT_ID) + .setEntityType(EntityType.K8S_POD.name()) + .setEntityName(entityName) + .putIdentifyingAttributes( + EntityConstants.getValue(CommonAttribute.COMMON_ATTRIBUTE_EXTERNAL_ID), + generateRandomUUIDAttrValue()) + .putAttributes( + attrName, + AttributeValue.newBuilder() + .setValue(Value.newBuilder().setString(attrValue).build()) + .build()) + .build(); + return entity; + } + private AttributeValue generateRandomUUIDAttrValue() { return AttributeValue.newBuilder() .setValue(Value.newBuilder()