From 8322f6f711eb69ac1d676bc76aaff8bda583e1c4 Mon Sep 17 00:00:00 2001 From: sangeetanadgir Date: Thu, 2 Feb 2023 17:31:56 +0530 Subject: [PATCH 1/3] ONECOND-2164-Add AppConfig support in Conductor --- .../conductor/core/config/CoreModule.java | 2 + .../core/execution/PropertiesLoader.java | 2 +- .../core/execution/WorkflowExecutor.java | 13 ++-- .../{MetaAppConfig.java => AppConfig.java} | 68 +++++++++++++++-- .../core/execution/appconfig/cache/Cache.java | 4 + .../netflix/conductor/dao/AppConfigDAO.java | 22 ++++++ .../netflix/conductor/dao/MetadataDAO.java | 7 -- .../server/resources/AdminResource.java | 74 +++++++++++++++++- .../conductor/aurora/AuroraAppConfigDAO.java | 52 +++++++++++++ .../conductor/aurora/AuroraMetadataDAO.java | 23 ------ .../migrations/V3_1__rename_config_store.sql | 14 ++++ server/build.gradle | 75 +++++++++++-------- .../conductor/server/ServerModule.java | 1 + 13 files changed, 278 insertions(+), 79 deletions(-) rename core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/{MetaAppConfig.java => AppConfig.java} (52%) create mode 100644 core/src/main/java/com/netflix/conductor/dao/AppConfigDAO.java create mode 100644 postgresql-persistence/src/main/java/com/netflix/conductor/aurora/AuroraAppConfigDAO.java create mode 100644 postgresql-persistence/src/main/resources/db/migrations/V3_1__rename_config_store.sql diff --git a/core/src/main/java/com/netflix/conductor/core/config/CoreModule.java b/core/src/main/java/com/netflix/conductor/core/config/CoreModule.java index ca57891667..124b44208a 100644 --- a/core/src/main/java/com/netflix/conductor/core/config/CoreModule.java +++ b/core/src/main/java/com/netflix/conductor/core/config/CoreModule.java @@ -23,6 +23,7 @@ import com.netflix.conductor.core.events.EventProcessor; import com.netflix.conductor.core.events.queue.dyno.DynoEventQueueProvider; import com.netflix.conductor.core.execution.WorkflowSweeper; +import com.netflix.conductor.core.execution.appconfig.cache.AppConfig; import com.netflix.conductor.core.execution.batch.BatchSweeper; import com.netflix.conductor.core.execution.batch.SherlockBatchProcessor; import com.netflix.conductor.core.execution.tasks.*; @@ -60,6 +61,7 @@ protected void configure() { bind(ErrorLookupTask.class).asEagerSingleton(); bind(PriorityLookupTask.class).asEagerSingleton(); bind(SetVariable.class).asEagerSingleton(); + bind(AppConfig.class).asEagerSingleton(); } } diff --git a/core/src/main/java/com/netflix/conductor/core/execution/PropertiesLoader.java b/core/src/main/java/com/netflix/conductor/core/execution/PropertiesLoader.java index bd423e07f6..494f03957e 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/PropertiesLoader.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/PropertiesLoader.java @@ -22,7 +22,7 @@ public PropertiesLoader(MetadataDAO metadata, @Named("properties") Map properties.put(it.getLeft(), StrSubstitutor.replace(it.getRight(), System.getenv()))); } diff --git a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java index 62813979d7..663464c1cb 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutor.java @@ -43,7 +43,7 @@ import com.netflix.conductor.core.events.queue.ObservableQueue; import com.netflix.conductor.core.execution.ApplicationException.Code; import com.netflix.conductor.core.execution.DeciderService.DeciderOutcome; -import com.netflix.conductor.core.execution.appconfig.cache.MetaAppConfig; +import com.netflix.conductor.core.execution.appconfig.cache.AppConfig; import com.netflix.conductor.core.execution.tasks.SubWorkflow; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.utils.IDGenerator; @@ -112,14 +112,14 @@ public class WorkflowExecutor { private final PropertiesLoader propertiesLoader; - private final MetaAppConfig metaAppConfig; + private final AppConfig appConfig; @Inject public WorkflowExecutor(MetadataDAO metadata, ExecutionDAO edao, QueueDAO queue, ErrorLookupDAO errorLookupDAO,ObjectMapper om, AuthManager auth, Configuration config, TaskStatusListener taskStatusListener, WorkflowStatusListener workflowStatusListener, - PropertiesLoader propertiesLoader, MetaAppConfig metaAppConfig) { + PropertiesLoader propertiesLoader, AppConfig appConfig) { this.metadata = metadata; this.edao = edao; this.queue = queue; @@ -136,7 +136,7 @@ public WorkflowExecutor(MetadataDAO metadata, ExecutionDAO edao, QueueDAO queue, this.authContextEnabled = Boolean.parseBoolean(config.getProperty("workflow.authcontext.enabled", "false")); this.lazyDecider = Boolean.parseBoolean(config.getProperty("workflow.lazy.decider", "false")); this.propertiesLoader = propertiesLoader; - this.metaAppConfig = metaAppConfig; + this.appConfig = appConfig; } public String startWorkflow(String name, int version, String correlationId, Map input) throws Exception { @@ -252,9 +252,8 @@ public String startWorkflow(String workflowId, String name, int version, Map configValues = new HashMap<>(); - configValues.put(MetaAppConfig.CC_EXTRACT_SERVER, metaAppConfig.getValue(MetaAppConfig.CC_EXTRACT_SERVER)); - configValues.put(MetaAppConfig.CHECKSUM_SERVER, metaAppConfig.getValue(MetaAppConfig.CHECKSUM_SERVER)); - configValues.put(MetaAppConfig.ONE_CDN_SERVER, metaAppConfig.getValue(MetaAppConfig.ONE_CDN_SERVER)); + Map configs = appConfig.getConfigs(); + configs.entrySet().forEach(x-> configValues.put(x.getKey(), x.getValue())); wf.setMetaConfigs(configValues); if (jobPriority == null) { diff --git a/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/MetaAppConfig.java b/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/AppConfig.java similarity index 52% rename from core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/MetaAppConfig.java rename to core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/AppConfig.java index cdde90fdc8..5f4849ceab 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/MetaAppConfig.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/AppConfig.java @@ -1,21 +1,28 @@ package com.netflix.conductor.core.execution.appconfig.cache; import com.google.inject.Inject; -import com.netflix.conductor.dao.MetadataDAO; +import com.netflix.conductor.dao.AppConfigDAO; import org.apache.commons.lang.text.StrSubstitutor; import org.apache.commons.lang3.tuple.Pair; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; import java.sql.SQLException; import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; /** * Class to obtain the Application specific Configuration values. */ -public class MetaAppConfig { +public class AppConfig { + + public static Logger logger; private Cache appCache; - private MetadataDAO metadataDAO; + + private AppConfigDAO appConfigDAO; private final static String APP_CACHE = "APP_CACHE"; private final int TTL_SECONDS = (int) TimeUnit.SECONDS.convert(1, TimeUnit.MINUTES); @@ -34,10 +41,12 @@ public class MetaAppConfig { @Inject - public MetaAppConfig(MetadataDAO metadataDAO) { + public AppConfig(AppConfigDAO appConfigDAO) { CacheManager cacheManager = CacheManager.getInstance(); appCache = cacheManager.getCache(APP_CACHE); - this.metadataDAO = metadataDAO; + this.appConfigDAO = appConfigDAO; + logger = LogManager.getLogger(AppConfig.class); + logger.info("Initialized AppConfig"); } /** @@ -50,13 +59,14 @@ public MetaAppConfig(MetadataDAO metadataDAO) { public String getValue(String key) throws Exception { String value; if ((value = appCache.get(key)) == null) { - synchronized (MetaAppConfig.class){ + synchronized (AppConfig.class){ if ((value = appCache.get(key)) == null) { reloadProperties(key); value = appCache.get(key); } } } + logger.info("AppConfig: Ask for " + key + ". Got " + value == null ? DEFAULT.get(key) : value); return value == null ? DEFAULT.get(key) : value; } @@ -71,6 +81,47 @@ public int getIntValue(String key) throws Exception { } + /** + * Method to obtain all the configs by querying the database + * + * @return + * @throws Exception + */ + public Map getConfigs() throws Exception { + if (appCache.getCurrentCache().isEmpty()) { + Map configValues = appConfigDAO.getConfigs(); + configValues.entrySet().forEach(configValue -> appCache.put(configValue.getKey(), StrSubstitutor.replace(configValue.getValue(), System.getenv()), TTL_SECONDS)); + return appCache.getCurrentCache(); + } else { + return appCache.getCurrentCache(); + } + } + + /** + * Method to store a new key/Value pair. + * This method will overwrite a value if the key already exists + * + * @param key + * @param value + * @throws Exception + */ + public void setValue(String key, String value) throws Exception { + appConfigDAO.setAppConfigValue(key, value); + appCache.invalidate(); + } + + /** + * Method to remove the configuration + * + * @param key + * @throws Exception + */ + public void removeConfig(String key) throws Exception { + appConfigDAO.removeAppConfigValue(key); + appCache.invalidate(); + } + + /** * Method to refresh the cache with the values from the database * @@ -80,8 +131,9 @@ public int getIntValue(String key) throws Exception { public synchronized void reloadProperties(String testKey) throws SQLException { if (appCache.get(testKey) == null) { appCache.invalidate(); - Pair configValue = metadataDAO.getConfigsByName(testKey); - appCache.put(configValue.getLeft(), StrSubstitutor.replace(configValue.getRight(), System.getenv()), TTL_SECONDS); + logger.info("AppConfig testKey " + testKey + ". Invalidating Cache "); + Map configValues = appConfigDAO.getConfigs(); + configValues.entrySet().forEach(configValue -> appCache.put(configValue.getKey(), StrSubstitutor.replace(configValue.getValue(), System.getenv()), TTL_SECONDS)); } } diff --git a/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/Cache.java b/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/Cache.java index 1461fa0ffa..46c5ad327f 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/Cache.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/Cache.java @@ -2,6 +2,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class Cache { @@ -62,6 +63,9 @@ public void expire() { cache.entrySet().removeIf(entry -> isExpired(nowSeconds, entry.getValue())); } + public Map getCurrentCache(){ + return cache.entrySet().stream().collect(Collectors.toMap(x-> x.getKey(), x-> (T) x.getValue().getData())); + } } diff --git a/core/src/main/java/com/netflix/conductor/dao/AppConfigDAO.java b/core/src/main/java/com/netflix/conductor/dao/AppConfigDAO.java new file mode 100644 index 0000000000..41285c14b2 --- /dev/null +++ b/core/src/main/java/com/netflix/conductor/dao/AppConfigDAO.java @@ -0,0 +1,22 @@ +package com.netflix.conductor.dao; + +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Collections; +import java.util.Map; + +public interface AppConfigDAO { + public default Pair getConfigsByName(String name){ + return null; + } + + public default Map getConfigs(){ + return Collections.emptyMap(); + } + + public default void setAppConfigValue(String key, String value){} + + public default void removeAppConfigValue(String key){} +} + + diff --git a/core/src/main/java/com/netflix/conductor/dao/MetadataDAO.java b/core/src/main/java/com/netflix/conductor/dao/MetadataDAO.java index 2fc1adc858..3fc49658bd 100644 --- a/core/src/main/java/com/netflix/conductor/dao/MetadataDAO.java +++ b/core/src/main/java/com/netflix/conductor/dao/MetadataDAO.java @@ -173,13 +173,6 @@ public default List> getConfigs() { return Collections.emptyList(); } - public default List> getConfigsByIsPreloaded(boolean isPreloaded) { - return Collections.emptyList(); - } - - public default Pair getConfigsByName(String name){ - return null; - } public default void addConfig(String name, String value) { } diff --git a/jersey/src/main/java/com/netflix/conductor/server/resources/AdminResource.java b/jersey/src/main/java/com/netflix/conductor/server/resources/AdminResource.java index 96fadb89df..2706c5f738 100644 --- a/jersey/src/main/java/com/netflix/conductor/server/resources/AdminResource.java +++ b/jersey/src/main/java/com/netflix/conductor/server/resources/AdminResource.java @@ -21,11 +21,14 @@ import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.core.config.Configuration; import com.netflix.conductor.core.execution.WorkflowExecutor; +import com.netflix.conductor.core.execution.appconfig.cache.AppConfig; import com.netflix.conductor.dao.MetadataDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.service.ExecutionService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +37,7 @@ import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import java.io.InputStream; +import java.sql.SQLException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,6 +52,7 @@ import com.netflix.conductor.core.execution.WorkflowExecutor; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.Response; /** * @author Viren @@ -70,13 +75,16 @@ public class AdminResource { private MetadataService metaservice; private WorkflowExecutor executor; + private AppConfig appConfig; + @Inject - public AdminResource(Configuration config, ExecutionService service, MetadataService metaservice,QueueDAO queue, MetadataDAO metadata,WorkflowExecutor executor) { + public AdminResource(Configuration config, ExecutionService service, MetadataService metaservice,QueueDAO queue, MetadataDAO metadata, AppConfig appConfig, WorkflowExecutor executor) { this.config = config; this.service = service; this.metaservice = metaservice; this.queue = queue; this.metadata = metadata; + this.appConfig = appConfig; this.version = "UNKNOWN"; this.buildDate = "UNKNOWN"; this.executor = executor; @@ -167,7 +175,6 @@ public void deleteConfig(@PathParam("name") String name, @Context HttpHeaders he @POST @Consumes({MediaType.WILDCARD}) - @Produces(MediaType.APPLICATION_JSON) @ApiOperation(value = "Reload configuration parameters from the database") @ApiImplicitParams({ @ApiImplicitParam(name = "Authorization", dataType = "string", paramType = "header")}) @@ -233,6 +240,69 @@ public HashMap getEnv( @QueryParam("keys") List keys){ } + @GET + @Path("/appconfig/key/{key}") + @Consumes(MediaType.TEXT_PLAIN) + @Produces({MediaType.TEXT_PLAIN}) + @ApiOperation(value = "Retrieves the App Config for the key ") + public String getAppConfig(@PathParam("key") String key, @Context HttpHeaders headers) { + + if (StringUtils.isEmpty(key)){ + return null; + } + try { + return appConfig.getValue(key); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @PUT + @Path("/appconfig/key/{key}/value/{value}") + @ApiOperation(value = "Adds a new config value to the database") + public void setAppConfig(@PathParam("key") String key, @PathParam("value") String value, @Context HttpHeaders headers) { + try { + appConfig.setValue(key, value); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @DELETE + @Path("/appconfig/key/{key}") + @ApiOperation(value = "Delete the App Config for the key") + public void deleteAppConfig(@PathParam("key") String key, @Context HttpHeaders headers) { + try { + appConfig.removeConfig(key); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + @GET + @Path("/appconfig/list") + @ApiOperation(value = "Get the list of all application configs from the database") + public Map getAppConfigs(@Context HttpHeaders headers) { + try { + return appConfig.getConfigs(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Path("/appconfig/refresh") + @ApiOperation(value = "Refresh the cache with list of App Configs from the database") + public void refreshAppConfig(@Context HttpHeaders headers) { + try { + appConfig.reloadProperties(""); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + private boolean bypassAuth(HttpHeaders headers) { if (!auth_referer_bypass) return false; diff --git a/postgresql-persistence/src/main/java/com/netflix/conductor/aurora/AuroraAppConfigDAO.java b/postgresql-persistence/src/main/java/com/netflix/conductor/aurora/AuroraAppConfigDAO.java new file mode 100644 index 0000000000..deb120368c --- /dev/null +++ b/postgresql-persistence/src/main/java/com/netflix/conductor/aurora/AuroraAppConfigDAO.java @@ -0,0 +1,52 @@ +package com.netflix.conductor.aurora; + +import com.netflix.conductor.core.config.Configuration; +import com.netflix.conductor.dao.AppConfigDAO; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.tuple.Pair; + +import javax.inject.Inject; +import javax.sql.DataSource; +import java.util.HashMap; +import java.util.Map; + +public class AuroraAppConfigDAO extends AuroraBaseDAO implements AppConfigDAO { + @Inject + public AuroraAppConfigDAO(DataSource dataSource, ObjectMapper mapper, Configuration config) { + super(dataSource, mapper); + } + + public Pair getConfigsByName(String key){ + final String SQL = "SELECT * FROM app_config where key = ?"; + return queryWithTransaction(SQL, q -> q.addParameter(key) + .executeAndFetchFirst(Pair.class)); + + } + + public Map getConfigs(){ + final String SQL = "SELECT key, value FROM app_config ORDER BY key, value"; + return queryWithTransaction(SQL, q -> q.executeAndFetch(rs -> { + Map configs = new HashMap<>(); + while (rs.next()) { + configs.put(rs.getString(1), rs.getString(2)); + } + return configs; + })); + } + + public void setAppConfigValue(String key, String value){ + final String SQL = "UPDATE app_config set value = ? where key = ?"; + queryWithTransaction(SQL, q -> q.addParameter(value) + .addParameter(key) + .executeUpdate()); + } + + public void removeAppConfigValue(String key, String value){ + final String SQL = "DELETE FROM app_config where key = ?"; + queryWithTransaction(SQL, q -> q.addParameter(key).executeUpdate()); + } + + + + +} diff --git a/postgresql-persistence/src/main/java/com/netflix/conductor/aurora/AuroraMetadataDAO.java b/postgresql-persistence/src/main/java/com/netflix/conductor/aurora/AuroraMetadataDAO.java index 3a3aacf0d8..77d0109702 100644 --- a/postgresql-persistence/src/main/java/com/netflix/conductor/aurora/AuroraMetadataDAO.java +++ b/postgresql-persistence/src/main/java/com/netflix/conductor/aurora/AuroraMetadataDAO.java @@ -281,29 +281,6 @@ public List> getConfigs() { })); } - public List> getConfigsByIsPreloaded(boolean isPreloaded) { - final String SQL = "SELECT name, value FROM config_store where is_preloaded = ?"; - return queryWithTransaction(SQL, q -> q.addParameter(isPreloaded).executeAndFetch(rs -> { - List> configs = new ArrayList<>(); - while (rs.next()) { - Pair entry = Pair.of(rs.getString(1), rs.getString(2)); - configs.add(entry); - } - return configs; - })); - } - - public Pair getConfigsByName(String name) { - final String SQL = "SELECT name, value FROM config_store where name = ?"; - return queryWithTransaction(SQL, q -> q.addParameter(name).executeAndFetch(rs -> { - Pair entry = null; - while (rs.next()) { - entry= Pair.of(rs.getString(1), rs.getString(2)); - } - return entry; - })); - } - @Override public void addConfig(String name, String value) { String SQL = "INSERT INTO meta_config (name, value) VALUES (?, ?) " + diff --git a/postgresql-persistence/src/main/resources/db/migrations/V3_1__rename_config_store.sql b/postgresql-persistence/src/main/resources/db/migrations/V3_1__rename_config_store.sql new file mode 100644 index 0000000000..1fb019132c --- /dev/null +++ b/postgresql-persistence/src/main/resources/db/migrations/V3_1__rename_config_store.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS config_store; + +create table if not exists app_config +( + key varchar(255) not null primary key, + value text +); + + +INSERT INTO app_config (key, value) VALUES ('cc_extract_server', 'http://nomad.service.${TLD}:4646') ON CONFLICT DO NOTHING; +INSERT INTO app_config (key, value) VALUES ('checksum_server', 'http://nomad.service.${TLD}:4646') ON CONFLICT DO NOTHING; +INSERT INTO app_config (key, value) VALUES ('one-cdn_server', 'http://nomad.service.${TLD}:4646') ON CONFLICT DO NOTHING; + + diff --git a/server/build.gradle b/server/build.gradle index 765a3a23c9..1dfe978c07 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -3,36 +3,22 @@ buildscript { classpath "org.akhikhl.gretty:gretty:1.2.4" } } - plugins { id 'com.github.johnrengelman.shadow' version '1.2.3' } - configurations.all { resolutionStrategy { force 'com.fasterxml.jackson.core:jackson-core:2.7.5' force 'com.fasterxml.jackson.core:jackson-databind:2.7.5' } } - apply plugin: 'war' apply plugin: "org.akhikhl.gretty" - dependencies { - configurations.all { - resolutionStrategy.eachDependency { details -> - if (details.requested.group == 'org.apache.logging.log4j' && details.target.version < '2.17.0') { - details.useVersion '2.17.0' - details.because 'CVE-2021-44228' - } - } - } - // Log4j compile 'org.apache.logging.log4j:log4j-core:2.9.0' compile 'org.apache.logging.log4j:log4j-jul:2.9.0' compile 'org.slf4j:jul-to-slf4j:1.7.25' - //Conductor compile project(':conductor-core') compile project(':conductor-jersey') @@ -40,26 +26,20 @@ dependencies { compile project(':conductor-contribs') compile project(':conductor-es6rest-persistence') compile project(':conductor-postgresql-persistence') - //Jetty - compile 'org.eclipse.jetty:jetty-server:9.3.9.v20160517' - compile 'org.eclipse.jetty:jetty-servlet:9.3.9.v20160517' - + compile 'org.eclipse.jetty:jetty-server:9.3.9.+' + compile 'org.eclipse.jetty:jetty-servlet:9.3.9.+' //Guice - compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' - compile 'com.google.inject:guice:4.1.0' - compile 'com.google.inject.extensions:guice-servlet:4.1.0' - + compile 'com.sun.jersey.contribs:jersey-guice:1.19.+' + compile 'com.google.inject:guice:4.+' + compile 'com.google.inject.extensions:guice-servlet:4.1.+' //Swagger compile 'io.swagger:swagger-jersey-jaxrs:1.5.0' - //In memory compile 'org.rarefiedredis.redis:redis-java:0.0.17' - // JNA library for EmbeddedElasticSearch compile 'net.java.dev.jna:jna:4.1.0' } - shadowJar { mergeServiceFiles() configurations = [project.configurations.compile] @@ -75,7 +55,6 @@ publishing { } } } - gretty { contextPath = '/' servletContainer = 'tomcat8' @@ -88,13 +67,47 @@ gretty { configurations.grettyRunnerTomcat8 { exclude group: 'org.slf4j', module: 'log4j-over-slf4j' } - - build.dependsOn('shadowJar') - task server(type: JavaExec) { - systemProperty 'loadSample', 'true' - systemProperty 'workflow.elasticsearch.url', 'localhost:9300' + environment 'APP_VERSION', '1.7.6-deluxe-server' + environment 'loadSample', 'false' + environment 'TLD', 'owf-dev' + environment 'STACK', 'test' + environment 'event_bus', 'nats_stream' + environment 'workflow_auth_validate', 'true' + environment 'workflow_authcontext_enabled', 'true' + environment 'workflow_failure_expandInline', 'false' + environment 'decider_sweep_frequency_seconds', '5' + environment 'workflow_sweeper_frequency', '2000' + environment 'workflow_sweeper_thread_count', '100' + environment 'workflow_event_processor_initial_delay', '5' + environment 'workflow_event_processor_refresh_seconds', '5' + environment 'workflow_system_task_worker_poll_frequency', '2000' + environment 'workflow_system_task_worker_queue_size', '1' + environment 'workflow_elasticsearch_mode', 'none' + environment 'workflow_elasticsearch_cluster_name', 'conductor.search' + environment 'workflow_elasticsearch_initial_sleep_seconds', '10' + environment 'workflow_elasticsearch_url', 'http://127.0.0.1:9200' + environment 'conductor_additional_modules', 'com.netflix.conductor.contribs.NatsStreamModule,com.netflix.conductor.contribs.ShotgunModule,com.netflix.conductor.contribs.AssetModule' + environment 'io_nats_streaming_clusterId', 'test-cluster' + environment 'io_nats_streaming_durableName', 'conductor-server-owf-dev' + environment 'io_nats_streaming_publishRetryIn', '5,10,15' + environment 'io_nats_streaming_url', 'nats://localhost:4222' + environment 'io_shotgun_dns', 'shotgun.service.owf-dev' + environment 'io_shotgun_service', 'conductor-server-owf-dev' + environment 'io_shotgun_publishRetryIn', '5,10,15' + environment 'conductor_auth_url', 'https://auth.dmlib.de/v1/tenant/deluxe/auth/token' + environment 'conductor_auth_clientId', 'deluxe.conductor' + environment 'conductor_auth_clientSecret', '...' + environment 'db', 'aurora' + environment 'aurora_db', 'local_conductor' + environment 'aurora_host', 'localhost' + environment 'aurora_port', '5432' + environment 'aurora_user', 'postgres' + environment 'aurora_password', 'deluxe123' + environment 'aurora_core_pool_size', '75' + environment 'aurora_core_connect_timeout', '30' main = 'com.netflix.conductor.server.Main' classpath = sourceSets.test.runtimeClasspath + setDebug(true) } \ No newline at end of file diff --git a/server/src/main/java/com/netflix/conductor/server/ServerModule.java b/server/src/main/java/com/netflix/conductor/server/ServerModule.java index ce8b6e7fda..658d881f41 100644 --- a/server/src/main/java/com/netflix/conductor/server/ServerModule.java +++ b/server/src/main/java/com/netflix/conductor/server/ServerModule.java @@ -161,6 +161,7 @@ private void configureAuroraContext() { bind(ExecutionDAO.class).to(AuroraExecutionDAO.class).asEagerSingleton(); bind(MetadataDAO.class).to(AuroraMetadataDAO.class).asEagerSingleton(); + bind(AppConfigDAO.class).to(AuroraAppConfigDAO.class).asEagerSingleton(); bind(QueueDAO.class).to(AuroraQueueDAO.class).asEagerSingleton(); bind(MetricsDAO.class).to(AuroraMetricsDAO.class).asEagerSingleton(); bind(IndexDAO.class).to(AuroraIndexDAO.class).asEagerSingleton(); From dcee9e5fae9f07ca1d49cc9b8e4ff79ca5df6f26 Mon Sep 17 00:00:00 2001 From: sangeetanadgir Date: Thu, 2 Feb 2023 17:36:19 +0530 Subject: [PATCH 2/3] ONECOND-2164 --- server/build.gradle | 75 +++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index 1dfe978c07..765a3a23c9 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -3,22 +3,36 @@ buildscript { classpath "org.akhikhl.gretty:gretty:1.2.4" } } + plugins { id 'com.github.johnrengelman.shadow' version '1.2.3' } + configurations.all { resolutionStrategy { force 'com.fasterxml.jackson.core:jackson-core:2.7.5' force 'com.fasterxml.jackson.core:jackson-databind:2.7.5' } } + apply plugin: 'war' apply plugin: "org.akhikhl.gretty" + dependencies { + configurations.all { + resolutionStrategy.eachDependency { details -> + if (details.requested.group == 'org.apache.logging.log4j' && details.target.version < '2.17.0') { + details.useVersion '2.17.0' + details.because 'CVE-2021-44228' + } + } + } + // Log4j compile 'org.apache.logging.log4j:log4j-core:2.9.0' compile 'org.apache.logging.log4j:log4j-jul:2.9.0' compile 'org.slf4j:jul-to-slf4j:1.7.25' + //Conductor compile project(':conductor-core') compile project(':conductor-jersey') @@ -26,20 +40,26 @@ dependencies { compile project(':conductor-contribs') compile project(':conductor-es6rest-persistence') compile project(':conductor-postgresql-persistence') + //Jetty - compile 'org.eclipse.jetty:jetty-server:9.3.9.+' - compile 'org.eclipse.jetty:jetty-servlet:9.3.9.+' + compile 'org.eclipse.jetty:jetty-server:9.3.9.v20160517' + compile 'org.eclipse.jetty:jetty-servlet:9.3.9.v20160517' + //Guice - compile 'com.sun.jersey.contribs:jersey-guice:1.19.+' - compile 'com.google.inject:guice:4.+' - compile 'com.google.inject.extensions:guice-servlet:4.1.+' + compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' + compile 'com.google.inject:guice:4.1.0' + compile 'com.google.inject.extensions:guice-servlet:4.1.0' + //Swagger compile 'io.swagger:swagger-jersey-jaxrs:1.5.0' + //In memory compile 'org.rarefiedredis.redis:redis-java:0.0.17' + // JNA library for EmbeddedElasticSearch compile 'net.java.dev.jna:jna:4.1.0' } + shadowJar { mergeServiceFiles() configurations = [project.configurations.compile] @@ -55,6 +75,7 @@ publishing { } } } + gretty { contextPath = '/' servletContainer = 'tomcat8' @@ -67,47 +88,13 @@ gretty { configurations.grettyRunnerTomcat8 { exclude group: 'org.slf4j', module: 'log4j-over-slf4j' } + + build.dependsOn('shadowJar') + task server(type: JavaExec) { - environment 'APP_VERSION', '1.7.6-deluxe-server' - environment 'loadSample', 'false' - environment 'TLD', 'owf-dev' - environment 'STACK', 'test' - environment 'event_bus', 'nats_stream' - environment 'workflow_auth_validate', 'true' - environment 'workflow_authcontext_enabled', 'true' - environment 'workflow_failure_expandInline', 'false' - environment 'decider_sweep_frequency_seconds', '5' - environment 'workflow_sweeper_frequency', '2000' - environment 'workflow_sweeper_thread_count', '100' - environment 'workflow_event_processor_initial_delay', '5' - environment 'workflow_event_processor_refresh_seconds', '5' - environment 'workflow_system_task_worker_poll_frequency', '2000' - environment 'workflow_system_task_worker_queue_size', '1' - environment 'workflow_elasticsearch_mode', 'none' - environment 'workflow_elasticsearch_cluster_name', 'conductor.search' - environment 'workflow_elasticsearch_initial_sleep_seconds', '10' - environment 'workflow_elasticsearch_url', 'http://127.0.0.1:9200' - environment 'conductor_additional_modules', 'com.netflix.conductor.contribs.NatsStreamModule,com.netflix.conductor.contribs.ShotgunModule,com.netflix.conductor.contribs.AssetModule' - environment 'io_nats_streaming_clusterId', 'test-cluster' - environment 'io_nats_streaming_durableName', 'conductor-server-owf-dev' - environment 'io_nats_streaming_publishRetryIn', '5,10,15' - environment 'io_nats_streaming_url', 'nats://localhost:4222' - environment 'io_shotgun_dns', 'shotgun.service.owf-dev' - environment 'io_shotgun_service', 'conductor-server-owf-dev' - environment 'io_shotgun_publishRetryIn', '5,10,15' - environment 'conductor_auth_url', 'https://auth.dmlib.de/v1/tenant/deluxe/auth/token' - environment 'conductor_auth_clientId', 'deluxe.conductor' - environment 'conductor_auth_clientSecret', '...' - environment 'db', 'aurora' - environment 'aurora_db', 'local_conductor' - environment 'aurora_host', 'localhost' - environment 'aurora_port', '5432' - environment 'aurora_user', 'postgres' - environment 'aurora_password', 'deluxe123' - environment 'aurora_core_pool_size', '75' - environment 'aurora_core_connect_timeout', '30' + systemProperty 'loadSample', 'true' + systemProperty 'workflow.elasticsearch.url', 'localhost:9300' main = 'com.netflix.conductor.server.Main' classpath = sourceSets.test.runtimeClasspath - setDebug(true) } \ No newline at end of file From a20238f3cccb2e05d3f51370e4d5180b5b372f03 Mon Sep 17 00:00:00 2001 From: sangeetanadgir Date: Thu, 2 Feb 2023 18:42:42 +0530 Subject: [PATCH 3/3] ONECOND-2164 --- .../conductor/core/execution/appconfig/cache/AppConfig.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/AppConfig.java b/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/AppConfig.java index 5f4849ceab..d12a5571d1 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/AppConfig.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/appconfig/cache/AppConfig.java @@ -88,13 +88,7 @@ public int getIntValue(String key) throws Exception { * @throws Exception */ public Map getConfigs() throws Exception { - if (appCache.getCurrentCache().isEmpty()) { - Map configValues = appConfigDAO.getConfigs(); - configValues.entrySet().forEach(configValue -> appCache.put(configValue.getKey(), StrSubstitutor.replace(configValue.getValue(), System.getenv()), TTL_SECONDS)); return appCache.getCurrentCache(); - } else { - return appCache.getCurrentCache(); - } } /**