[deployement] make kubernetes yamls for aws operational (apache#6192)

### Motivation

Supplied Kubernetes yaml's for AWS are outdated and just don't work.

### Modifications

Update yaml files and so that appying them on AWS EKS will actually set up a working Pulsar environment.

### Verifying this change

This change is a trivial rework / code cleanup without any test coverage.

deployment/kubernetes/aws/bookkeeper.yaml

@@ -21,13 +21,12 @@
 
 # SSDs for bookie journal storage
 kind: StorageClass
-apiVersion: storage.k8s.io/v1beta1
+apiVersion: storage.k8s.io/v1
 metadata:
   name: bookie-ssd
 provisioner: kubernetes.io/aws-ebs
 parameters:
   type: gp2
-  zones: us-east-2a
   fsType: xfs
 #  To create encrypted ebs volume using kms
 #  encrypted: "true"
@@ -36,13 +35,12 @@ reclaimPolicy: Retain
 ---
 # HDDs for bookie ledger storage
 kind: StorageClass
-apiVersion: storage.k8s.io/v1beta1
+apiVersion: storage.k8s.io/v1
 metadata:
   name: bookie-hdd
 provisioner: kubernetes.io/aws-ebs
 parameters:
   type: st1
-  zones: us-east-2a
   fsType: xfs
 #  To create encrypted ebs volume using kms
 #  encrypted: "true"
@@ -54,14 +52,15 @@ kind: ConfigMap
 metadata:
   name: bookie-config
 data:
-  PULSAR_MEM: "\"-Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintHeapAtGC -verbosegc -XX:G1LogLevel=finest -Xms28g -Xmx28g -XX:MaxDirectMemorySize=28g\""
-  PULSAR_PREFIX_dbStorage_writeCacheMaxSizeMb: "2048" # Write cache size (direct memory)
-  PULSAR_PREFIX_dbStorage_readAheadCacheMaxSizeMb: "2048" # Read cache size (direct memory)
-  PULSAR_PREFIX_dbStorage_rocksDB_blockCacheSize: "4294967296"
-  PULSAR_PREFIX_journalMaxSizeMB: "2048"
-  PULSAR_PREFIX_zkServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
-  PULSAR_PREFIX_statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
-  PULSAR_PREFIX_useHostNameAsBookieID: "true"
+  BOOKIE_MEM: "\"-Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintHeapAtGC -verbosegc -XX:G1LogLevel=finest -Xms28g -Xmx28g -XX:MaxDirectMemorySize=28g\""
+  BOOKIE_GC: "\"-XX:+UseG1GC -XX:MaxGCPauseMillis=10\""
+  dbStorage_writeCacheMaxSizeMb: "2048" # Write cache size (direct memory)
+  dbStorage_readAheadCacheMaxSizeMb: "2048" # Read cache size (direct memory)
+  dbStorage_rocksDB_blockCacheSize: "4294967296"
+  journalMaxSizeMB: "2048"
+  zkServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+  statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
+  useHostNameAsBookieID: "true"
 ---
 apiVersion: apps/v1beta1
 kind: StatefulSet
@@ -203,9 +202,9 @@ spec:
                         name: bookie-config
                 env:
                     ## Configure for lower mem usage
-                  - name: PULSAR_MEM
+                  - name: BOOKIE_MEM
                     value: "\" -Xmx256m \""
-                  - name: PULSAR_GC
+                  - name: BOOKIE_GC
                     value: "\"  \""
 # Uncomment the following if bookie should be deployed on a specific instance type
 #            nodeSelector:

deployment/kubernetes/aws/broker.yaml

@@ -27,13 +27,13 @@ data:
     # better GC behavior at high throughput
     PULSAR_MEM: "\" -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -Xms12g -Xmx12g -XX:MaxDirectMemorySize=14g -Dpulsar.root.logger=DEBUG,FILE \""
     PULSAR_GC: "\" -XX:+UseG1GC -XX:MaxGCPauseMillis=10\""
-    PULSAR_PREFIX_zookeeperServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
-    PULSAR_PREFIX_configurationStoreServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
-    PULSAR_PREFIX_clusterName: us-east
-    PULSAR_PREFIX_managedLedgerDefaultEnsembleSize: "2"
-    PULSAR_PREFIX_managedLedgerDefaultWriteQuorum: "2"
-    PULSAR_PREFIX_managedLedgerDefaultAckQuorum: "2"
-    PULSAR_PREFIX_deduplicationEnabled: "false"
+    zookeeperServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+    configurationStoreServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+    clusterName: us-east
+    managedLedgerDefaultEnsembleSize: "2"
+    managedLedgerDefaultWriteQuorum: "2"
+    managedLedgerDefaultAckQuorum: "2"
+    deduplicationEnabled: "false"
 ---
 ##
 ## Broker deployment definition
@@ -63,7 +63,7 @@ spec:
 #            dnsPolicy: ClusterFirstWithHostNet
             containers:
               - name: broker
-                image: apachepulsar/pulsar:1.20.0-incubating
+                image: apachepulsar/pulsar:latest
                 command: ["sh", "-c"]
                 args:
                   - >
@@ -127,7 +127,7 @@ metadata:
 spec:
     containers:
       - name: pulsar-admin
-        image: apachepulsar/pulsar:1.20.0-incubating
+        image: apachepulsar/pulsar:latest
         command: ["sh", "-c"]
         args:
           - >

deployment/kubernetes/aws/cluster-metadata.yaml

@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: pulsar-cluster-metadata-init
+  labels:
+    app: pulsar
+    component: broker
+spec:
+  template:
+    spec:
+      containers:
+        - name: pulsar-cluster-metadata-init-container
+          image: apachepulsar/pulsar:latest
+          command: ["sh", "-c"]
+          args:
+            - >
+              bin/pulsar initialize-cluster-metadata \
+                --cluster us-east \
+                --zookeeper zookeeper \
+                --configuration-store zookeeper \
+                --web-service-url http://broker:8080/ \
+                --broker-service-url pulsar://broker:6650/ || true;
+      restartPolicy: Never

deployment/kubernetes/aws/proxy.yaml

@@ -23,10 +23,11 @@ metadata:
   name: pulsar-proxy-config
 data:
   PULSAR_MEM: "\" -Xms4g -Xmx4g -XX:MaxDirectMemorySize=4g\""
-  PULSAR_PREFIX_brokerServiceURL: pulsar://broker:6650
-  PULSAR_PREFIX_brokerWebServiceURL: http://broker:8080
-  PULSAR_PREFIX_clusterName: pulsar-eks
-
+  brokerServiceURL: pulsar://broker:6650
+  brokerWebServiceURL: http://broker:8080
+  clusterName: us-east
+  zookeeperServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+  configurationStoreServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
 ---
 apiVersion: apps/v1beta1
 kind: Deployment