Add support for indexing to postgres persistence (Netflix#181)

persistence/README.md

@@ -27,7 +27,7 @@ Group: `com.netflix.conductor`
 ```properties
 conductor.db.type=mysql
 
-#Cache expiry for teh task definitions in seconds
+#Cache expiry for the task definitions in seconds
 conductor.mysql.taskDefCacheRefreshInterval=60
 
 #Use spring datasource properties to configure MySQL connection
@@ -45,8 +45,6 @@ spring.datasource.hikari.auto-commit=
 
 ```properties
 conductor.db.type=postgres
-#Cache expiry for teh task definitions in seconds
-conductor.mysql.taskDefCacheRefreshInterval=60
 
 #Use spring datasource properties to configure Postgres connection
 spring.datasource.url=
@@ -55,3 +53,30 @@ spring.datasource.password=
 spring.datasource.hikari.maximum-pool-size=
 spring.datasource.hikari.auto-commit=
 ```
+
+Additionally, the postgres module includes the ability to index your workflow and task executions and to store task execution logs in Postgres without requiring ElasticSearch.
+
+This can be enabled by setting the following in your application properties file:
+
+```properties
+conductor.indexing.type=postgres
+conductor.indexing.enabled=true
+# The following is to force Elastic Search IndexDAO not to run. If it just missing it will still try to start v6
+conductor.elasticsearch.version=postgres
+```
+
+It supports full querying of logs through the UI, and exposes the Postgres full text search in two ways. If you search for a chunk of JSON:
+
+```JSON
+{"workflowType":"my_workflow", "version": 3}
+```
+
+It will use an index to search for documents matching the values in the JSON. In this example, all JSON docs with a `workflowType` attribute set to `my_workflow` and a `version` attribute set to `3`.
+
+You can also use a full text search on the whole unstructured JSON document. This uses the Postgres [tsquery](https://www.postgresql.org/docs/11/datatype-textsearch.html#DATATYPE-TSQUERY) syntax for constructing searches. For example:
+
+```
+my-correlation-id & my-workflow
+```
+
+Will search for any document containing both `my-correlation-id` and `my-workflow`.

persistence/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresConfiguration.java

@@ -24,16 +24,14 @@
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.DependsOn;
-import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.*;
 import org.springframework.retry.RetryContext;
 import org.springframework.retry.backoff.NoBackOffPolicy;
 import org.springframework.retry.policy.SimpleRetryPolicy;
 import org.springframework.retry.support.RetryTemplate;
 
 import com.netflix.conductor.postgres.dao.PostgresExecutionDAO;
+import com.netflix.conductor.postgres.dao.PostgresIndexDAO;
 import com.netflix.conductor.postgres.dao.PostgresMetadataDAO;
 import com.netflix.conductor.postgres.dao.PostgresQueueDAO;
 
@@ -95,6 +93,15 @@ public PostgresQueueDAO postgresQueueDAO(
         return new PostgresQueueDAO(retryTemplate, objectMapper, dataSource);
     }
 
+    @Bean
+    @DependsOn({"flywayForPrimaryDb"})
+    @Conditional(PostgresIndexConditions.PostgresIndexingEnabled.class)
+    public PostgresIndexDAO postgresIndexDAO(
+            @Qualifier("postgresRetryTemplate") RetryTemplate retryTemplate,
+            ObjectMapper objectMapper) {
+        return new PostgresIndexDAO(retryTemplate, objectMapper, dataSource);
+    }
+
     @Bean
     public RetryTemplate postgresRetryTemplate(PostgresProperties properties) {
         SimpleRetryPolicy retryPolicy = new CustomRetryPolicy();

persistence/postgres-persistence/src/main/java/com/netflix/conductor/postgres/config/PostgresIndexConditions.java

@@ -0,0 +1,41 @@
+/*
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.postgres.config;
+
+import org.springframework.boot.autoconfigure.condition.AllNestedConditions;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+
+public class PostgresIndexConditions {
+
+    private PostgresIndexConditions() {}
+
+    public static class PostgresIndexingEnabled extends AllNestedConditions {
+
+        PostgresIndexingEnabled() {
+            super(ConfigurationPhase.PARSE_CONFIGURATION);
+        }
+
+        @SuppressWarnings("unused")
+        @ConditionalOnProperty(
+                name = "conductor.indexing.enabled",
+                havingValue = "true",
+                matchIfMissing = true)
+        static class enabledIndexing {}
+
+        @SuppressWarnings("unused")
+        @ConditionalOnProperty(
+                name = "conductor.indexing.type",
+                havingValue = "postgres",
+                matchIfMissing = true)
+        static class enabledPostgresIndexing {}
+    }
+}