Merge pull request #288 from jembi/CU-86bygn3jn_Images-Placement-for-…

…Client-Registry-Mapper-Kafka-FHIR Cu 86bygn3jn images placement for client registry mapper kafka fhir
jembi · May 31, 2024 · ec6ae59 · ec6ae59
2 parents 4eca587 + 193d376
commit ec6ae59
Show file tree

Hide file tree

Showing 25 changed files with 96 additions and 74 deletions.
diff --git a/client-registry-jempi/docker-compose.combined-cluster.yml b/client-registry-jempi/docker-compose.combined-cluster.yml
@@ -1,4 +1,4 @@
-version: '3.9'
+version: "3.9"
 
 services:
   jempi-bootstrapper:

diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml
@@ -1,4 +1,4 @@
-version: '3.9'
+version: "3.9"
 
 services:
   jempi-async-receiver:
@@ -21,7 +21,6 @@ services:
       default:
       jempi:
 
-
   jempi-etl:
     image: jembi/jempi-etl:${JEMPI_ETL_IMAGE_TAG}
     environment:
@@ -40,7 +39,6 @@ services:
       default:
       jempi:
 
-
   jempi-controller:
     image: jembi/jempi-controller:${JEMPI_CONTROLLER_IMAGE_TAG}
     environment:
@@ -75,7 +73,6 @@ services:
       jempi:
       postgres:
 
-
   jempi-linker:
     image: jembi/jempi-linker:${JEMPI_LINKER_IMAGE_TAG}
     environment:
@@ -111,7 +108,6 @@ services:
       jempi:
       postgres:
 
-
   jempi-bootstrapper:
     image: jembi/jempi-bootstrapper:${JEMPI_BOOTSTRAPPER_IMAGE_TAG}
     environment:
@@ -134,11 +130,9 @@ services:
       jempi:
       postgres:
 
-
 volumes:
   jempi-shared-data:
 
-
 networks:
   kafka:
     name: kafka_public

diff --git a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml
@@ -1,24 +1,24 @@
-version: '3.9'
+version: "3.9"
 
 services:
   jempi-zero-01:
     deploy:
       placement:
         constraints:
-          - node.labels.name == node-1
+          - node.labels.name == ${JEMPI_ZERO_01_PLACEMENT}
     command: dgraph zero --my=jempi-zero-01:5080 --replicas 3 --bindall --raft "idx=1"
     networks:
       jempi:
 
   jempi-zero-02:
-    image: dgraph/dgraph:v22.0.0
+    image: ${JEMPI_ZERO_IMAGE}
     volumes:
       - jempi-zero-02-data:/dgraph
     deploy:
       replicas: 1
       placement:
         constraints:
-          - node.labels.name == node-2
+          - node.labels.name == ${JEPMI_ZERO_02_PLACEMENT}
       resources:
         limits:
           memory: ${JEMPI_ZERO_MEMORY_LIMIT}
@@ -31,14 +31,14 @@ services:
       jempi:
 
   jempi-zero-03:
-    image: dgraph/dgraph:v22.0.0
+    image: ${JEMPI_ZERO_IMAGE}
     volumes:
       - jempi-zero-03-data:/dgraph
     deploy:
       replicas: 1
       placement:
         constraints:
-          - node.labels.name == node-3
+          - node.labels.name == ${JEMPI_ZERO_03_PLACEMENT}
       resources:
         limits:
           memory: ${JEMPI_ZERO_MEMORY_LIMIT}

diff --git a/client-registry-jempi/docker-compose.dgraph-zero.yml b/client-registry-jempi/docker-compose.dgraph-zero.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   jempi-zero-01:
-    image: dgraph/dgraph:v22.0.0
+    image: ${JEMPI_ZERO_IMAGE}
     volumes:
       - jempi-zero-01-data:/dgraph
     deploy:

diff --git a/client-registry-jempi/docker-compose.dgraph.yml b/client-registry-jempi/docker-compose.dgraph.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   jempi-alpha-01:
-    image: dgraph/dgraph:v22.0.0
+    image: ${JEMPI_ZERO_IMAGE}
     hostname: alpha-01
     volumes:
       - jempi-alpha-01-data:/dgraph
@@ -20,7 +20,7 @@ services:
       jempi:
 
   jempi-alpha-02:
-    image: dgraph/dgraph:v22.0.0
+    image: ${JEMPI_ZERO_IMAGE}
     hostname: alpha-02
     volumes:
       - jempi-alpha-02-data:/dgraph
@@ -38,7 +38,7 @@ services:
       jempi:
 
   jempi-alpha-03:
-    image: dgraph/dgraph:v22.0.0
+    image: ${JEMPI_ZERO_IMAGE}
     hostname: alpha-03
     volumes:
       - jempi-alpha-03-data:/dgraph
@@ -56,7 +56,7 @@ services:
       jempi:
 
   jempi-ratel:
-    image: dgraph/ratel:v21.03.2
+    image: ${JEMPI_RATEL_IMAGE}
     deploy:
       replicas: 1
       resources:

diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json
@@ -12,8 +12,14 @@
   ],
   "environmentVariables": {
     "KAFKA_HOSTS": "kafka-01:9092",
+    "JEMPI_POSTGRESQL_IMAGE": "bitnami/postgresql-repmgr:15.2.0",
+    "JEMPI_ZERO_IMAGE": "dgraph/dgraph:v22.0.0",
+    "JEMPI_RATEL_IMAGE": "dgraph/ratel:v21.03.2",
     "JEMPI_ZERO_MEMORY_LIMIT": "3G",
     "JEMPI_ZERO_MEMORY_RESERVE": "500M",
+    "JEMPI_ZERO_01_PLACEMENT": "node-1",
+    "JEMPI_ZERO_02_PLACEMENT": "node-2",
+    "JEMPI_ZERO_03_PLACEMENT": "node-3",
     "JEMPI_ALPHA_01_MEMORY_LIMIT": "3G",
     "JEMPI_ALPHA_01_MEMORY_RESERVE": "500M",
     "JEMPI_ALPHA_02_MEMORY_LIMIT": "3G",

diff --git a/client-registry-santempi/docker-compose.yml b/client-registry-santempi/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   santedb-mpi:
-    image: santesuite/santedb-mpi:2.2.3
+    image: ${SANTEMPI_IMAGE}
     networks:
       mpi:
       public:
@@ -27,7 +27,7 @@ services:
       - santedb-data:/santedb
 
   santedb-www:
-    image: santesuite/santedb-www:2.2.1.10
+    image: ${SANTEDB_WWW_IMAGE}
     networks:
       default:
       reverse-proxy:

diff --git a/client-registry-santempi/package-metadata.json b/client-registry-santempi/package-metadata.json
@@ -7,8 +7,16 @@
   "dependencies": ["database-postgres"],
   "environmentVariables": {
     "SANTEMPI_INSTANCES": "1",
-    "POSTGRESQL_PASSWORD": "instant101",
-    "POSTGRESQL_USERNAME": "postgres",
-    "SANTEMPI_DB_REP_SET": "postgres-1"
+    "SANTE_POSTGRES_PASSWORD": "postgres",
+    "SANTEMPI_POSTGRESQL_PASSWORD": "SanteDB123",
+    "SANTEMPI_POSTGRESQL_USERNAME": "santedb",
+    "SANTEMPI_REPMGR_PRIMARY_HOST": "santempi-psql-1",
+    "SANTEMPI_REPMGR_PARTNER_NODES": "santempi-psql-1",
+    "SANTEMPI_IMAGE": "santesuite/santedb-mpi:2.2.3",
+    "SANTEDB_WWW_IMAGE": "santesuite/santedb-www:2.2.1.10",
+    "SANTEMPI_PSQL_IMAGE": "bitnami/postgresql-repmgr:14",
+    "PSQL_1_PLACEMENT": "node-1",
+    "PSQL_2_PLACEMENT": "node-2",
+    "PSQL_3_PLACEMENT": "node-3"
   }
 }
diff --git a/data-mapper-logstash/docker-compose.yml b/data-mapper-logstash/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   data-mapper-logstash:
-    image: docker.elastic.co/logstash/logstash:7.12.1
+    image: ${LOGSTASH_IMAGE}
     healthcheck:
       test: curl --fail http://localhost:9600 || exit 1
       interval: 10s
@@ -19,7 +19,7 @@ services:
       - logstash-data:/usr/share/logstash/data
     deploy:
       placement:
-        max_replicas_per_node: 1
+        max_replicas_per_node: ${LOGSTASH_MAX_REPLICAS_PER_NODE}
       replicas: ${LOGSTASH_INSTANCES}
       resources:
         limits:

diff --git a/data-mapper-logstash/package-metadata.json b/data-mapper-logstash/package-metadata.json
@@ -6,7 +6,9 @@
   "version": "0.0.1",
   "dependencies": ["analytics-datastore-elastic-search"],
   "environmentVariables": {
+    "LOGSTASH_IMAGE": "docker.elastic.co/logstash/logstash:7.12.1",
     "LOGSTASH_INSTANCES": "1",
+    "LOGSTASH_MAX_REPLICAS_PER_NODE": "1",
     "LS_JAVA_OPTS": "-Xmx2g -Xms2g",
     "LOGSTASH_YML_CONFIG": "logstash-logstash.yml",
     "LOGSTASH_DEV_MOUNT": "false",

diff --git a/database-postgres/docker-compose-postgres.cluster.yml b/database-postgres/docker-compose-postgres.cluster.yml
@@ -10,7 +10,7 @@ services:
           - "node.labels.name==${POSTGRES_1_PLACEMENT}"
 
   postgres-2:
-    image: bitnami/postgresql-repmgr:14
+    image: ${POSTGRES_IMAGE}
     environment:
       POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
       REPMGR_NODE_NETWORK_NAME: postgres-2
@@ -45,7 +45,7 @@ services:
       postgres:
 
   postgres-3:
-    image: bitnami/postgresql-repmgr:14
+    image: ${POSTGRES_IMAGE}
     environment:
       POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
       REPMGR_NODE_NETWORK_NAME: postgres-3

diff --git a/database-postgres/docker-compose-postgres.yml b/database-postgres/docker-compose-postgres.yml
@@ -20,7 +20,7 @@ services:
       - target: /bitnami/postgresql/conf/conf.d/custom_postgresql.conf
         source: postgresql.conf
     deploy:
-      replicas: 1
+      replicas: ${POSTGRES_REPLICAS}
       resources:
         limits:
           cpus: ${POSTGRES_CPU_LIMIT}

diff --git a/database-postgres/package-metadata.json b/database-postgres/package-metadata.json
@@ -35,6 +35,7 @@
     "PGPOOL_MEMORY_RESERVE": "50M",
     "HF_PGPOOL_ENABLED": "true",
     "POSTGRES_FAILOVER": "automatic",
-    "POSTGRES_DEGRADED_MONITORING_TIMEOUT": "5"
+    "POSTGRES_DEGRADED_MONITORING_TIMEOUT": "5",
+    "POSTGRES_REPLICAS": "1"
   }
 }
diff --git a/kafka-mapper-consumer/docker-compose.yml b/kafka-mapper-consumer/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '3.9'
+version: "3.9"
 
 services:
   kafka-mapper-consumer:

diff --git a/kafka-unbundler-consumer/docker-compose.yml b/kafka-unbundler-consumer/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   kafka-unbundler-consumer:
-    image: jembi/kafka-unbundler-consumer:0.0.3
+    image: ${KAFKA_UNBUNDLER_CONSUMER_IMAGE}
     environment:
       KAFKA_HOST: ${KAFKA_HOST}
       KAFKA_PORT: ${KAFKA_PORT}

diff --git a/kafka-unbundler-consumer/package-metadata.json b/kafka-unbundler-consumer/package-metadata.json
@@ -6,6 +6,7 @@
   "version": "0.0.1",
   "dependencies": ["message-bus-kafka"],
   "environmentVariables": {
+    "KAFKA_UNBUNDLER_CONSUMER_IMAGE": "jembi/kafka-unbundler-consumer:0.0.3",
     "KAFKA_HOST": "kafka-01",
     "KAFKA_PORT": "9092"
   }

diff --git a/message-bus-helper-hapi-proxy/docker-compose.yml b/message-bus-helper-hapi-proxy/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   hapi-proxy:
-    image: jembi/springproxyserver:20211217-170809.3f57ef5
+    image: ${HAPI_PROXY_IMAGE}
     hostname: hapi-proxy
     environment:
       HAPI_SERVER_URL: ${HAPI_SERVER_URL}

diff --git a/message-bus-helper-hapi-proxy/package-metadata.json b/message-bus-helper-hapi-proxy/package-metadata.json
@@ -6,6 +6,7 @@
   "type": "infrastructure",
   "dependencies": ["message-bus-kafka", "fhir-datastore-hapi-fhir"],
   "environmentVariables": {
+    "HAPI_PROXY_IMAGE": "jembi/springproxyserver:20211217-170809.3f57ef5",
     "HAPI_PROXY_INSTANCES": 1,
     "HAPI_PROXY_CPU_LIMIT": "0",
     "HAPI_PROXY_CPU_RESERVE": "0.05",