JeMPI Bootstrapper app

JeMPI_Apps/JeMPI_Bootstrapper/README.md

@@ -0,0 +1,75 @@
+The app fall under the devops part of JeMPI, and is currently used to manage data used by JeMPI. More specifically, it is used to manage JeMPI postgres, draph, and kafka data. It manages this data through a CLI interface of which the options are list below:
+
+./bootstrapper.sh data -h
+```
+Usage: <main class> data [-hV] [-c=<config>] [COMMAND]
+  -c, --config=<config>   Config file
+  -h, --help              Show this help message and exit.
+  -V, --version           Print version information and exit.
+Commands:
+  kafka
+  dgraph
+  postgres
+  resetAll             Deletes all data and schemas associated with JeMPI, then
+                         recreates schemas, and add initial data.
+  deleteAllSchemaData  Delete all the data and schema used by JeMPI.
+  createAllSchemaData  Create all the required schema's and data for JeMPI.
+```
+
+./bootstrapper.sh data kafka -h
+```
+Usage: <main class> data kafka [-hV] [COMMAND]
+  -h, --help      Show this help message and exit.
+  -V, --version   Print version information and exit.
+Commands:
+  resetAll             Deletes all data and schemas associated with JeMPI kafka
+                         instance, then recreates schemas, and add initial data.
+  deleteAll            Delete all the data and schema used by JeMPI kafka
+                         instance.
+  createAllSchemaData  Create all the required schema's and data for JeMPI
+                         Kafka instance.
+  listTopics           List all the topics associated with the JeMPI instance.
+  describeTopic        Describe a topic associated with the JeMPI instance.
+```
+
+./bootstrapper.sh data postgres -h
+```
+Usage: <main class> data postgres [-hV] [COMMAND]
+  -h, --help      Show this help message and exit.
+  -V, --version   Print version information and exit.
+Commands:
+  resetAll             Deletes all data and schemas associated with JeMPI
+                         Postgres instance, then recreates schemas, and add
+                         initial data.
+  deleteDataOnly       Delete all the data (only) used by JeMPI Postgres
+                         instance.
+  deleteAll            Delete all the data and schema used by JeMPI Postgres
+                         instance.
+  createAllSchemaData  Create all the required schema's and data for JeMPI
+                         Postgres instance.
+```
+
+./bootstrapper.sh data dgraph -h
+```
+Usage: <main class> data dgraph [-hV] [COMMAND]
+  -h, --help      Show this help message and exit.
+  -V, --version   Print version information and exit.
+Commands:
+  resetAll             Deletes all data and schemas associated with JeMPI
+                         Dgraph instance, then recreates schemas, and add
+                         initial data.
+  deleteAll            Delete all the data and schema used by JeMPI Dgraph
+                         instance.
+  createAllSchemaData  Create all the required schema's and data for JeMPI
+                         Dgraph instance.
+```
+
+
+**Other notes**
+
+- This application can be run directly (as a java app), or via the script found at devops/linux/bootstrapper.sh (i.e `devops/linux/bootstrapper.sh -h`)
+
+- The app uses the JeMPI environment variable to know what to connect to for the various instances. You can however pass in a config file that contains the variables you want to use instead. These variables in the config file will then be merged with the available environment variables.
+    - A sample to the config format can be found here (JeMPI_Apps/JeMPI_Bootstrapper/boostrap.conf.sample)
+    - To use this config file you need to specify the config option (i.e `./bootstrapper data resetAll config="<path-to-config>"`)
+

JeMPI_Apps/JeMPI_Bootstrapper/boostrap.conf.sample

@@ -0,0 +1,9 @@
+POSTGRESQL_IP=127.0.0.1
+POSTGRESQL_PORT=5432
+POSTGRESQL_USER=postgres
+POSTGRESQL_PASSWORD=
+POSTGRESQL_PASSWORD=jempi
+KAFKA_BOOTSTRAP_SERVERS=127.0.0.1
+KAFKA_APPLICATION_ID=aId
+DGRAPH_HOSTS=127.0.0.1
+DGRAPH_PORTS=5080

JeMPI_Apps/JeMPI_Bootstrapper/pom.xml

@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.jembi.jempi</groupId>
+        <artifactId>JeMPI</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>JeMPI_Bootstrapper</artifactId>
+    <packaging>jar</packaging>
+
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <start-class>${project.groupId}.bootstrapper.BootstrapperCLI</start-class>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>info.picocli</groupId>
+            <artifactId>picocli</artifactId>
+            <version>4.6.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.jembi.jempi</groupId>
+            <artifactId>JeMPI_LibShared</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.typesafe</groupId>
+            <artifactId>config</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.vavr</groupId>
+            <artifactId>vavr</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.dgraph</groupId>
+            <artifactId>dgraph4j</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-jcl</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <version>RELEASE</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>5.10.0</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jembi.jempi</groupId>
+            <artifactId>JeMPI_LibMPI</artifactId>
+            <version>1.0-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.eclipse.m2e</groupId>
+                    <artifactId>lifecycle-mapping</artifactId>
+                    <version>1.0.0</version>
+                    <configuration>
+                        <lifecycleMappingMetadata>
+                            <pluginExecutions>
+                                <pluginExecution>
+                                    <pluginExecutionFilter>
+                                        <groupId>org.apache.maven.plugins</groupId>
+                                        <artifactId>maven-enforcer-plugin</artifactId>
+                                        <versionRange>[1.0.0,)</versionRange>
+                                        <goals>
+                                            <goal>enforce</goal>
+                                        </goals>
+                                    </pluginExecutionFilter>
+                                    <action>
+                                        <ignore/>
+                                    </action>
+                                </pluginExecution>
+                            </pluginExecutions>
+                        </lifecycleMappingMetadata>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+
+    </build>
+
+</project>

JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/Bootstrapper.java

@@ -0,0 +1,13 @@
+package org.jembi.jempi.bootstrapper;
+
+import org.apache.logging.log4j.Logger;
+import org.jembi.jempi.bootstrapper.utils.BootstrapperLogger;
+
+public class Bootstrapper {
+    protected static final Logger LOGGER = BootstrapperLogger.getLogger("Jempi Bootstrapper");
+    protected BootstrapperConfig loadedConfig;
+    public Bootstrapper(final String configFilePath) {
+        this.loadedConfig = BootstrapperConfig.create(configFilePath, LOGGER);
+    }
+}
+

JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/BootstrapperCLI.java

@@ -0,0 +1,22 @@
+package org.jembi.jempi.bootstrapper;
+import org.jembi.jempi.bootstrapper.data.cli.CLI;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+@Command(
+        mixinStandardHelpOptions = true,
+        subcommands = {
+        CLI.class,
+})
+
+public class BootstrapperCLI implements Runnable {
+
+    @CommandLine.Option(names = {"-c", "--config"}, description = "Config file")
+    private String config;
+    @Override
+    public void run() { }
+
+    public static void main(String... args) {
+        int exitCode = new CommandLine(new BootstrapperCLI()).execute(args);
+        System.exit(exitCode);
+    }
+}

JeMPI_Apps/JeMPI_Bootstrapper/src/main/java/org/jembi/jempi/bootstrapper/BootstrapperConfig.java

@@ -0,0 +1,100 @@
+package org.jembi.jempi.bootstrapper;
+
+import java.io.File;
+import java.util.Arrays;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.logging.log4j.Logger;
+
+public class BootstrapperConfig {
+
+    public final String POSTGRESQL_IP;
+    public final Integer POSTGRESQL_PORT;
+    public final String POSTGRESQL_USER ;
+    public final String POSTGRESQL_PASSWORD;
+    public final String POSTGRESQL_DATABASE;
+    public final String KAFKA_BOOTSTRAP_SERVERS;
+    public final String KAFKA_APPLICATION_ID;
+    public final String[] DGRAPH_ALPHA_HOSTS;
+    public final int[] DGRAPH_ALPHA_PORTS;
+
+    public BootstrapperConfig(Config parsedConfig) {
+        POSTGRESQL_IP = parsedConfig.getString("POSTGRESQL_IP");
+        POSTGRESQL_PORT = parsedConfig.getInt("POSTGRESQL_PORT");
+        POSTGRESQL_USER = parsedConfig.getString("POSTGRESQL_USER");
+        POSTGRESQL_PASSWORD = parsedConfig.getString("POSTGRESQL_PASSWORD");
+        POSTGRESQL_DATABASE = parsedConfig.getString("POSTGRESQL_DATABASE");
+        KAFKA_BOOTSTRAP_SERVERS = parsedConfig.getString("KAFKA_BOOTSTRAP_SERVERS");
+        KAFKA_APPLICATION_ID = parsedConfig.getString("KAFKA_APPLICATION_ID");
+        DGRAPH_ALPHA_HOSTS = parsedConfig.getString("DGRAPH_HOSTS").split(",");
+        DGRAPH_ALPHA_PORTS = Arrays.stream(parsedConfig.getString("DGRAPH_PORTS").split(",")).mapToInt(s -> {
+            try {
+                return Integer.parseInt(s);
+            } catch (NumberFormatException ex) {
+                return Integer.MIN_VALUE;
+            }
+        }).toArray();
+    }
+    public static BootstrapperConfig create(String filepath, Logger LOGGER) {
+        return new BootstrapperConfig(new Builder(LOGGER)
+                .withOptionalFile(filepath)
+                .withSystemEnvironment()
+                .withSystemProperties()
+                .build());
+    }
+
+    private static class Builder {
+
+        private static final Config SYSTEM_PROPERTIES = ConfigFactory.systemProperties();
+        private static final Config SYSTEM_ENVIRONMENT = ConfigFactory.systemEnvironment();
+
+        private Config conf = ConfigFactory.empty();
+
+        private final Logger logger;
+        public Builder(Logger logger) {
+            this.logger = logger;
+        }
+
+        // This should return the current executing user path
+        private static String getExecutionDirectory() {
+            return SYSTEM_PROPERTIES.getString("user.dir");
+        }
+
+        Builder withSystemProperties() {
+            conf = conf.withFallback(SYSTEM_PROPERTIES);
+            return this;
+        }
+
+        Builder withSystemEnvironment() {
+            conf = conf.withFallback(SYSTEM_ENVIRONMENT);
+            return this;
+        }
+
+        Builder withOptionalFile(final String path) {
+            if (path == null){
+                return this;
+            }
+            File secureConfFile = new File(path);
+
+            if (!secureConfFile.isAbsolute()) {
+                secureConfFile = new File(getExecutionDirectory() + path);
+            }
+            if (secureConfFile.exists()) {
+                this.logger.info("Loaded config file from path ({})", path);
+                conf = conf.withFallback(ConfigFactory.parseFile(secureConfFile));
+            } else {
+                this.logger.info("Attempted to load file from path ({}) but it was not found", path);
+            }
+            return this;
+        }
+
+        Config build() {
+            conf = conf.resolve();
+            return conf;
+        }
+
+    }
+
+}
+
+