Skip to content

Commit ea06271

Browse files
sanket-mundrasanket-mundra
authored
update max string length for bulk update matching all filter api (#277)
* update max string length for bulk update matching all filter api * upgrade vulneable version * increase expiry
1 parent 2fdd761 commit ea06271

File tree

10 files changed

+75
-52
lines changed

10 files changed

+75
-52
lines changed

‎entity-service-change-event-generator/build.gradle.kts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ dependencies {
1919
implementation("org.glassfish.jersey.core:jersey-common:2.34") {
2020
because("https://snyk.io/vuln/SNYK-JAVA-ORGGLASSFISHJERSEYCORE-1255637")
2121
}
22-
implementation("org.apache.commons:commons-compress:1.21") {
22+
implementation("org.apache.commons:commons-compress:1.26") {
2323
because("Multiple vulnerabilities")
2424
}
2525
runtimeOnly("org.jetbrains.kotlin:kotlin-stdlib:1.6.21") {
@@ -28,7 +28,7 @@ dependencies {
2828
}
2929

3030
runtimeOnly("io.confluent:kafka-protobuf-serializer")
31-
implementation(platform("org.hypertrace.core.kafkastreams.framework:kafka-bom:0.4.2"))
31+
implementation(platform("org.hypertrace.core.kafkastreams.framework:kafka-bom:0.4.7"))
3232

3333
annotationProcessor("org.projectlombok:lombok:1.18.18")
3434
compileOnly("org.projectlombok:lombok:1.18.18")

‎entity-service-factory/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ dependencies {
88
implementation(project(":entity-service-attribute-translator"))
99
implementation(project(":entity-service-impl"))
1010
implementation(project(":entity-service-change-event-generator"))
11-
implementation("org.hypertrace.core.documentstore:document-store:0.7.55")
11+
implementation("org.hypertrace.core.documentstore:document-store:0.7.58")
1212
}

‎entity-service-impl/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ dependencies {
1212
annotationProcessor("org.projectlombok:lombok:1.18.22")
1313
compileOnly("org.projectlombok:lombok:1.18.18")
1414

15-
implementation("org.hypertrace.core.documentstore:document-store:0.7.55")
15+
implementation("org.hypertrace.core.documentstore:document-store:0.7.58")
1616
implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.12.6")
1717
implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.12.6")
1818
implementation("org.hypertrace.core.attribute.service:caching-attribute-service-client:0.14.15")

‎entity-service-impl/src/main/java/org/hypertrace/entity/query/service/EntityQueryServiceImpl.java‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ public class EntityQueryServiceImpl extends EntityQueryServiceImplBase {
113113
private static final DocumentParser DOCUMENT_PARSER = new DocumentParser();
114114
private static final String CHUNK_SIZE_CONFIG = "entity.query.service.response.chunk.size";
115115
private static final String ENTITY_IDS_DELETE_LIMIT_CONFIG = "entity.delete.limit";
116+
private static final String MAX_STRING_LENGTH_FOR_UPDATE_CONFIG =
117+
"entity.query.service.max.string.length.for.update";
116118

117119
private static final int DEFAULT_CHUNK_SIZE = 10_000;
118120
private static final String ARRAY_VALUE_PATH_SUFFIX =
@@ -125,7 +127,6 @@ public class EntityQueryServiceImpl extends EntityQueryServiceImplBase {
125127
.findFieldByNumber(VALUES_FIELD_NUMBER)
126128
.getJsonName())
127129
.collect(joining("."));
128-
private static final int MAX_STRING_LENGTH_FOR_UPDATE = 1000;
129130

130131
private final Collection entitiesCollection;
131132
private final EntityQueryConverter entityQueryConverter;
@@ -138,6 +139,7 @@ public class EntityQueryServiceImpl extends EntityQueryServiceImplBase {
138139
private final EntityAttributeChangeEvaluator entityAttributeChangeEvaluator;
139140
private final EntityCounterMetricSender entityCounterMetricSender;
140141
private final EntityNormalizer entityNormalizer;
142+
private final int maxStringLengthForUpdate;
141143

142144
public EntityQueryServiceImpl(
143145
Datastore datastore,
@@ -159,7 +161,10 @@ public EntityQueryServiceImpl(
159161
: config.getInt(CHUNK_SIZE_CONFIG),
160162
config.hasPath(ENTITY_IDS_DELETE_LIMIT_CONFIG)
161163
? config.getInt(ENTITY_IDS_DELETE_LIMIT_CONFIG)
162-
: 10000);
164+
: 10000,
165+
config.hasPath(MAX_STRING_LENGTH_FOR_UPDATE_CONFIG)
166+
? config.getInt(MAX_STRING_LENGTH_FOR_UPDATE_CONFIG)
167+
: 5000);
163168
}
164169

165170
public EntityQueryServiceImpl(
@@ -171,7 +176,8 @@ public EntityQueryServiceImpl(
171176
EntityCounterMetricSender entityCounterMetricSender,
172177
Channel entityTypeChannel,
173178
int chunkSize,
174-
int maxEntitiesToDelete) {
179+
int maxEntitiesToDelete,
180+
int maxStringLengthForUpdate) {
175181
this(
176182
entitiesCollection,
177183
datastore,
@@ -182,7 +188,8 @@ public EntityQueryServiceImpl(
182188
new EntityFetcher(entitiesCollection, DOCUMENT_PARSER),
183189
entityTypeChannel,
184190
chunkSize,
185-
maxEntitiesToDelete);
191+
maxEntitiesToDelete,
192+
maxStringLengthForUpdate);
186193
}
187194

188195
EntityQueryServiceImpl(
@@ -195,7 +202,8 @@ public EntityQueryServiceImpl(
195202
EntityFetcher entityFetcher,
196203
Channel entityTypeChannel,
197204
int chunkSize,
198-
int maxEntitiesToDelete) {
205+
int maxEntitiesToDelete,
206+
int maxStringLengthForUpdate) {
199207
this.entitiesCollection = entitiesCollection;
200208
this.entityAttributeMapping = entityAttributeMapping;
201209
this.entityQueryConverter = new EntityQueryConverter(entityAttributeMapping);
@@ -210,6 +218,7 @@ public EntityQueryServiceImpl(
210218
IdentifyingAttributeCache identifyingAttributeCache = new IdentifyingAttributeCache(datastore);
211219
this.entityNormalizer =
212220
new EntityNormalizer(entityTypeClient, new EntityIdGenerator(), identifyingAttributeCache);
221+
this.maxStringLengthForUpdate = maxStringLengthForUpdate;
213222
}
214223

215224
@Override
@@ -745,12 +754,12 @@ public void bulkUpdateAllMatchingFilter(
745754
if (anyStringUpdateViolatesLengthConstraint(request)) {
746755
LOG.warn(
747756
String.format(
748-
"String update value exceeded %d characters", MAX_STRING_LENGTH_FOR_UPDATE));
757+
"String update value exceeded %d characters", this.maxStringLengthForUpdate));
749758
responseObserver.onError(
750759
Status.INVALID_ARGUMENT
751760
.withDescription(
752761
String.format(
753-
"Update value too long (> %d characters)", MAX_STRING_LENGTH_FOR_UPDATE))
762+
"Update value too long (> %d characters)", this.maxStringLengthForUpdate))
754763
.asException());
755764
}
756765

@@ -991,7 +1000,7 @@ private boolean anyStringUpdateViolatesLengthConstraint(
9911000
.map(LiteralConstant::getValue)
9921001
.flatMap(this::getStringStream)
9931002
.map(String::length)
994-
.anyMatch(length -> length > MAX_STRING_LENGTH_FOR_UPDATE);
1003+
.anyMatch(length -> length > this.maxStringLengthForUpdate);
9951004
}
9961005

9971006
private Stream<String> getStringStream(final Value value) {

‎entity-service-impl/src/test/java/org/hypertrace/entity/query/service/EntityQueryServiceImplTest.java‎

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ public void testUpdate_noTenantId() throws Exception {
137137
entityFetcher,
138138
mock(Channel.class),
139139
1,
140-
1000);
140+
1000,
141+
5000);
141142

142143
eqs.update(null, mockResponseObserver);
143144

@@ -167,7 +168,8 @@ public void testUpdate_noEntityType() throws Exception {
167168
entityFetcher,
168169
mock(Channel.class),
169170
1,
170-
1000);
171+
1000,
172+
5000);
171173

172174
eqs.update(EntityUpdateRequest.newBuilder().build(), mockResponseObserver);
173175

@@ -198,7 +200,8 @@ public void testUpdate_noEntityId() throws Exception {
198200
entityFetcher,
199201
mock(Channel.class),
200202
1,
201-
1000);
203+
1000,
204+
5000);
202205

203206
eqs.update(
204207
EntityUpdateRequest.newBuilder().setEntityType(TEST_ENTITY_TYPE).build(),
@@ -231,7 +234,8 @@ public void testUpdate_noOperation() throws Exception {
231234
entityFetcher,
232235
mock(Channel.class),
233236
1,
234-
1000);
237+
1000,
238+
5000);
235239

236240
eqs.update(
237241
EntityUpdateRequest.newBuilder()
@@ -293,7 +297,8 @@ public void testUpdate_success() throws Exception {
293297
entityFetcher,
294298
mock(Channel.class),
295299
1,
296-
1000);
300+
1000,
301+
5000);
297302
eqs.update(updateRequest, mockResponseObserver);
298303
return null;
299304
});
@@ -330,7 +335,8 @@ public void testBulkUpdate_noTenantId() throws Exception {
330335
entityFetcher,
331336
mock(Channel.class),
332337
1,
333-
1000);
338+
1000,
339+
5000);
334340

335341
eqs.bulkUpdate(null, mockResponseObserver);
336342

@@ -361,7 +367,8 @@ public void testBulkUpdate_noEntityType() throws Exception {
361367
entityFetcher,
362368
mock(Channel.class),
363369
1,
364-
1000);
370+
1000,
371+
5000);
365372

366373
eqs.bulkUpdate(BulkEntityUpdateRequest.newBuilder().build(), mockResponseObserver);
367374

@@ -392,7 +399,8 @@ public void testBulkUpdate_noEntities() throws Exception {
392399
entityFetcher,
393400
mock(Channel.class),
394401
1,
395-
1000);
402+
1000,
403+
5000);
396404

397405
eqs.bulkUpdate(
398406
BulkEntityUpdateRequest.newBuilder().setEntityType(TEST_ENTITY_TYPE).build(),
@@ -432,7 +440,8 @@ public void testBulkUpdate_entitiesWithNoUpdateOperations() throws Exception {
432440
entityFetcher,
433441
mock(Channel.class),
434442
1,
435-
1000);
443+
1000,
444+
5000);
436445
eqs.bulkUpdate(bulkUpdateRequest, mockResponseObserver);
437446
return null;
438447
});
@@ -478,7 +487,8 @@ public void testBulkUpdate_success() throws Exception {
478487
entityFetcher,
479488
mock(Channel.class),
480489
1,
481-
1000);
490+
1000,
491+
5000);
482492
eqs.bulkUpdate(bulkUpdateRequest, mockResponseObserver);
483493
return null;
484494
});
@@ -518,7 +528,8 @@ void testBulkUpdateAllMatchingFilter_noTenantId() throws Exception {
518528
entityFetcher,
519529
mock(Channel.class),
520530
1,
521-
1000);
531+
1000,
532+
5000);
522533

523534
eqs.bulkUpdateAllMatchingFilter(null, mockResponseObserver);
524535

@@ -551,7 +562,8 @@ void testBulkUpdateAllMatchingFilter_noEntityType() throws Exception {
551562
entityFetcher,
552563
mock(Channel.class),
553564
1,
554-
1000);
565+
1000,
566+
5000);
555567

556568
eqs.bulkUpdateAllMatchingFilter(
557569
BulkUpdateAllMatchingFilterRequest.newBuilder().build(), mockResponseObserver);
@@ -588,7 +600,8 @@ void testBulkUpdateAllMatchingFilter_entitiesWithNoUpdateOperations() throws Exc
588600
entityFetcher,
589601
mock(Channel.class),
590602
1,
591-
1000);
603+
1000,
604+
5000);
592605
eqs.bulkUpdateAllMatchingFilter(bulkUpdateRequest, mockResponseObserver);
593606

594607
verify(mockResponseObserver, times(1))
@@ -687,7 +700,8 @@ void testBulkUpdateAllMatchingFilter_success() throws Exception {
687700
entityFetcher,
688701
mock(Channel.class),
689702
1,
690-
1000);
703+
1000,
704+
5000);
691705
eqs.bulkUpdateAllMatchingFilter(bulkUpdateRequest, mockResponseObserver);
692706
return null;
693707
});
@@ -726,7 +740,8 @@ public void testExecute_noTenantId() throws Exception {
726740
entityFetcher,
727741
mock(Channel.class),
728742
1,
729-
1000);
743+
1000,
744+
5000);
730745

731746
eqs.execute(null, mockResponseObserver);
732747

@@ -802,7 +817,8 @@ public void testExecute_success() throws Exception {
802817
new EntityCounterMetricSender(),
803818
mock(Channel.class),
804819
1,
805-
1000);
820+
1000,
821+
5000);
806822

807823
eqs.execute(request, mockResponseObserver);
808824
return null;
@@ -893,7 +909,8 @@ public void testExecute_success_chunksize_2() throws Exception {
893909
new EntityCounterMetricSender(),
894910
mock(Channel.class),
895911
2,
896-
1000);
912+
1000,
913+
5000);
897914

898915
eqs.execute(request, mockResponseObserver);
899916
return null;
@@ -947,7 +964,8 @@ void testBulkUpdateEntityArrayAttribute() throws Exception {
947964
entityFetcher,
948965
mock(Channel.class),
949966
1,
950-
1000);
967+
1000,
968+
5000);
951969
eqs.bulkUpdateEntityArrayAttribute(request, mockResponseObserver);
952970
return null;
953971
});
@@ -1020,7 +1038,8 @@ public void testDeleteEntities() throws IOException {
10201038
entityFetcher,
10211039
mock(Channel.class),
10221040
100,
1023-
1000);
1041+
1000,
1042+
5000);
10241043

10251044
eqs.deleteEntities(request, mockResponseObserver);
10261045
});
@@ -1093,7 +1112,8 @@ public void testExecute_withAliases() throws Exception {
10931112
entityFetcher,
10941113
mock(Channel.class),
10951114
100,
1096-
1000);
1115+
1000,
1116+
5000);
10971117

10981118
eqs.execute(request, mockResponseObserver);
10991119
});
@@ -1140,7 +1160,8 @@ public void test_buildTotalQuery() throws Exception {
11401160
entityFetcher,
11411161
mock(Channel.class),
11421162
1,
1143-
1000);
1163+
1000,
1164+
5000);
11441165
StreamObserver<TotalEntitiesResponse> mockResponseObserver = mock(StreamObserver.class);
11451166

11461167
Context.current()
@@ -1186,7 +1207,8 @@ public void test_sendCorrectTotalResponse() throws Exception {
11861207
entityFetcher,
11871208
mock(Channel.class),
11881209
1,
1189-
1000);
1210+
1000,
1211+
5000);
11901212
StreamObserver<TotalEntitiesResponse> mockResponseObserver = mock(StreamObserver.class);
11911213

11921214
when(entitiesCollection.total(any())).thenReturn(123L);

‎entity-service/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ dependencies {
2323
implementation(project(":entity-service-factory"))
2424

2525
implementation("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.62")
26-
implementation("org.hypertrace.core.documentstore:document-store:0.7.55")
26+
implementation("org.hypertrace.core.documentstore:document-store:0.7.58")
2727

2828
runtimeOnly("io.grpc:grpc-netty:1.57.2")
2929

‎entity-service/src/integrationTest/resources/configs/entity-service/application.conf‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,10 @@ entity.service.config = {
2323
publish.change.events = false
2424
}
2525

26-
entity.query.service.response.chunk.size = 2
26+
entity.query.service = {
27+
response.chunk.size = 2
28+
max.string.length.for.update = 5000
29+
}
2730
eqs.change.notification.enabled.entity.types = ["*"]
2831
attribute.service.config = {
2932
host = localhost

‎helm/templates/entity-service-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ data:
4444
schema.registry.url = "http://schema-registry-service:8081"
4545
}
4646
}
47+
entity.query.service.max.string.length.for.update = {{ .Values.entityQueryService.maxStringLengthForUpdate }}
4748
{{- if .Values.attributes }}
4849
entity.service.attributeMap = [
4950
{{- range $i, $v := .Values.attributes }}

‎helm/values.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,3 +145,6 @@ hpa:
145145
minReplicas: 1
146146
maxReplicas: 5
147147
targetCPUUtilizationPercentage: 80
148+
149+
entityQueryService:
150+
maxStringLengthForUpdate: 5000

0 commit comments

Comments
 (0)