Document store config overhaul (#254)

Author: suresh-prakash

entity-service-factory/src/main/java/org/hypertrace/entity/service/EntityServiceDataStoreConfig.java

@@ -1,29 +1,21 @@
 package org.hypertrace.entity.service;
 
 import com.typesafe.config.Config;
-import org.hypertrace.core.documentstore.DocumentStoreConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.hypertrace.core.documentstore.model.config.DatastoreConfig;
+import org.hypertrace.core.documentstore.model.config.TypesafeConfigDatastoreConfigExtractor;
 
 public class EntityServiceDataStoreConfig {
 
-  private static final Logger LOG = LoggerFactory.getLogger(EntityServiceDataStoreConfig.class);
+  private static final String DATASTORE_TYPE_CONFIG = "dataStoreType";
 
-  private final String dataStoreType;
-
-  private final Config entityServiceConfig;
+  private final Config documentStoreConfig;
 
   public EntityServiceDataStoreConfig(Config config) {
-    entityServiceConfig = config.getConfig("entity.service.config.entity-service");
-    dataStoreType = entityServiceConfig.getString(DocumentStoreConfig.DATASTORE_TYPE_CONFIG_KEY);
-  }
-
-  public String getDataStoreType() {
-    return dataStoreType;
+    documentStoreConfig = config.getConfig("entity.service.config.document.store");
   }
 
-  public Config getDataStoreConfig() {
-    entityServiceConfig.entrySet().forEach(e -> LOG.info("Config key {}", e.getKey()));
-    return entityServiceConfig.getConfig(getDataStoreType());
+  public DatastoreConfig getDataStoreConfig() {
+    return TypesafeConfigDatastoreConfigExtractor.from(documentStoreConfig, DATASTORE_TYPE_CONFIG)
+        .extract();
   }
 }

entity-service-factory/src/main/java/org/hypertrace/entity/service/EntityServiceFactory.java

@@ -35,10 +35,11 @@ public List<GrpcPlatformService> buildServices(
       GrpcServiceContainerEnvironment grpcServiceContainerEnvironment) {
     this.grpcServiceContainerEnvironment = grpcServiceContainerEnvironment;
     Config config = grpcServiceContainerEnvironment.getConfig(SERVICE_NAME);
-    EntityServiceDataStoreConfig dataStoreConfig = new EntityServiceDataStoreConfig(config);
-    this.datastore =
-        DatastoreProvider.getDatastore(
-            dataStoreConfig.getDataStoreType(), dataStoreConfig.getDataStoreConfig());
+    EntityServiceDataStoreConfig entityServiceDatastoreConfig =
+        new EntityServiceDataStoreConfig(config);
+    datastore = DatastoreProvider.getDatastore(entityServiceDatastoreConfig.getDataStoreConfig());
+    grpcServiceContainerEnvironment.getLifecycle().shutdownComplete().thenRun(datastore::close);
+
     EntityAttributeMapping entityAttributeMapping =
         new EntityAttributeMapping(config, grpcServiceContainerEnvironment.getChannelRegistry());
     EntityChangeEventGenerator entityChangeEventGenerator =

entity-service/src/integrationTest/java/org/hypertrace/entity/service/service/EntityQueryServiceTest.java

@@ -146,9 +146,6 @@ public class EntityQueryServiceTest {
   private static final String ATTRIBUTE_SERVICE_HOST_KEY = "attribute.service.config.host";
   private static final String ATTRIBUTE_SERVICE_PORT_KEY = "attribute.service.config.port";
 
-  private static final String MONGO_HOST_KEY = "entity.service.config.entity-service.mongo.host";
-  private static final String MONGO_PORT_KEY = "entity.service.config.entity-service.mongo.port";
-
   private static final Config config = getServiceConfig();
 
   @BeforeAll
@@ -2943,13 +2940,7 @@ private Expression createExpression(String columnName) {
 
   private static Datastore getDatastore() {
     EntityServiceDataStoreConfig entityServiceConfig = new EntityServiceDataStoreConfig(config);
-
-    Map<String, String> mongoConfig = new HashMap<>();
-    mongoConfig.putIfAbsent("host", config.getString(MONGO_HOST_KEY));
-    mongoConfig.putIfAbsent("port", config.getString(MONGO_PORT_KEY));
-    Config dataStoreConfig = ConfigFactory.parseMap(mongoConfig);
-    String dataStoreType = entityServiceConfig.getDataStoreType();
-    return DatastoreProvider.getDatastore(dataStoreType, dataStoreConfig);
+    return DatastoreProvider.getDatastore(entityServiceConfig.getDataStoreConfig());
   }
 
   private static Map<String, Map<String, String>> getAttributesMap() {

entity-service/src/integrationTest/resources/configs/entity-service/application.conf

@@ -3,15 +3,23 @@ service.name = entity-service
 service.port = 50061
 service.admin.port = 50062
 entity.service.config = {
-  entity-service {
+  document.store {
     dataStoreType = mongo
+    appName = entity-service-integration-test
+    maxPoolSize = 10
     mongo {
-      host = localhost
-      host = ${?MONGO_HOST}
-      port = 37017
-      port = ${?MONGO_PORT}
+      database = default_db
+      endpoints = [
+        {
+          host = localhost
+          host = ${?MONGO_HOST}
+          port = 37017
+          port = ${?MONGO_PORT}
+        }
+      ]
     }
   }
+
   publish.change.events = false
 }
 

entity-service/src/main/resources/configs/common/application.conf

@@ -3,12 +3,22 @@ service.name = entity-service
 service.port = 50061
 service.admin.port = 50062
 entity.service.config = {
-  entity-service {
+  document.store {
     dataStoreType = mongo
+    appName = entity-service-local
+    maxPoolSize = 10
     mongo {
-      host = localhost
-      host = ${?MONGO_HOST}
-      port = 27017
+      database = default_db
+      user = ${?MONGO_SERVICE_USERNAME}
+      password = ${?MONGO_SERVICE_USER_PASSWORD}
+      endpoints = [
+        {
+          host = localhost
+          host = ${?MONGO_HOST} # provides a way to override the mongo_host via an environment variable
+          port = 27017
+          port = ${?MONGO_PORT}
+        }
+      ]
     }
   }
   publish.change.events = false

helm/templates/deployment.yaml

@@ -83,6 +83,20 @@ spec:
               value: "/var/{{ .Chart.Name }}/log/log4j2.properties"
             - name: JAVA_OPTS
               value: {{ .Values.javaOpts | quote }}
+            {{- if and .Values.database .Values.database.mongo.authEnabled }}
+            {{- with .Values.database.mongo.credentials }}
+            - name: MONGO_SERVICE_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .secretName }}
+                  key: {{ .secretUsernameKey }}
+            - name: MONGO_SERVICE_USER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .secretName }}
+                  key: {{ .secretPasswordKey }}
+            {{- end }}
+            {{- end }}
           volumeMounts:
             - name: service-config
               mountPath: /app/resources/configs/{{ .Chart.Name }}/application.conf

helm/templates/entity-service-config.yaml

@@ -7,20 +7,24 @@ metadata:
 data:
   application.conf: |-
     entity.service.config = {
-      entity-service {
-        {{- $dst := .Values.entityServiceConfig.dataStoreType }}
+      document.store {
+        {{- if .Values.database }}
+        {{- $dst := .Values.database.type }}
         dataStoreType = {{ $dst }}
+        appName = {{ .Values.service.name }}
+
         {{ $dst }} {
-          {{- range $key, $value := (index .Values "entityServiceConfig" (printf "%s" $dst)) }}
+          {{- range $key, $value := (index .Values "database" (printf "%s" $dst)) }}
           {{- if $value }}
-            {{- if eq $key "password" }}
+            {{- if hasPrefix "${?" (printf "%s" $value) }}
           {{ $key }} = {{ $value }}
             {{- else }}
-          {{ $key }} = {{ $value | quote }}
+          {{ $key }} = {{- toJson $value }}
             {{- end }}
           {{- end }}
           {{- end }}
         }
+        {{- end }}
       }
       publish.change.events = {{ .Values.entityServiceConfig.publishChangeEvents }}
     }

helm/values.yaml

@@ -92,10 +92,6 @@ serviceSelectorLabels:
 entityServiceConfig:
   name: entity-service-config
   publishChangeEvents: false
-  dataStoreType: "mongo"
-  mongo:
-    host: mongo
-    url: ""
   postgres:
     host: postgres
     port: 5432
@@ -106,6 +102,20 @@ entityServiceConfig:
     host: attribute-service
     port: 9012
 
+database:
+  type: mongo
+  mongo:
+    authEnabled: true
+    endpoints:
+      - host: mongo
+        port: 27017
+    replicaSet: rs0
+    authDb: admin
+    credentials:
+      secretName: mongodb-credentials
+      secretUsernameKey: mongodb-service-username
+      secretPasswordKey: mongodb-service-password
+
 attributes: []
 idAttributes:
   - scope: API