MySQL server deployment (#707)

* MYSQL server deployment

* Add mysql in allroles

* New private ip for mysql

* genconfig mysql

* deploy.py connect mysql

* updatemysql

* mysql deployment yaml

* get node lists for service - mysql

* Take the first element

* Update

src/ClusterBootstrap/az_params.py

@@ -2,16 +2,19 @@
     "azure_cluster" : {
         "subscription": "Bing DLTS",
         "infra_node_num": 1, 
-        "worker_node_num": 2, 
+        "worker_node_num": 2,
+        "mysql_node_num": 0,
         "nfs_node_num": 1,
         "azure_location": "westus2",
         "infra_vm_size" : "Standard_D1_v2",
         "worker_vm_size": "Standard_NC6",
+        "mysql_vm_size": "Standard_D1_v2",
         "nfs_vm_size": "Standard_D1_v2",
         "vm_image" : "Canonical:UbuntuServer:18.04-LTS:18.04.201907221",
         "vm_local_storage_sku" : "Premium_LRS",
         "infra_local_storage_sz" : 1023,
         "worker_local_storage_sz" : 1023,
+        "mysql_local_storage_sz": 2048,
         "nfs_data_disk_sku" : "Premium_LRS",
         "nfs_data_disk_sz" : 1023,
         "nfs_data_disk_num": 1,

src/ClusterBootstrap/az_tools.py

@@ -113,7 +113,7 @@ def create_vm(vmname, vm_ip, role, vm_size, pwd, vmcnf):
     else:
         auth = """--generate-ssh-keys --authentication-type ssh --ssh-key-value '%s' """ % config["azure_cluster"]["sshkey"]
 
-    priv_IP = "--private-ip-address %s " % vm_ip if not role in ["worker","nfs"] else ""
+    priv_IP = "--private-ip-address %s " % vm_ip if not role in ["worker", "mysql", "nfs"] else ""
     nsg = "nfs_nsg_name" if role == "nfs" else "nsg_name"
 
     availability_set = ""
@@ -125,7 +125,7 @@ def create_vm(vmname, vm_ip, role, vm_size, pwd, vmcnf):
         assert os.path.exists(config["cloud_init_%s" % role])
         cloud_init = "--custom-data {}".format(config["cloud_init_%s" % role])
 
-    if role in ["infra", "worker"]:		
+    if role in ["infra", "worker", "mysql"]:
         storage = "--storage-sku {} --data-disk-sizes-gb {} ".format(config["azure_cluster"]["vm_local_storage_sku"],
                 config["azure_cluster"]["%s_local_storage_sz" % role])
         # corner case: NFS on infra
@@ -489,6 +489,11 @@ def create_cluster(arm_vm_password=None, parallelism=1):
 
     add_workers(arm_vm_password, parallelism)
 
+    # create mysql if specified
+    for i in range(int(config["azure_cluster"]["mysql_node_num"])):
+        create_vm_param(i, "mysql", config["azure_cluster"]["mysql_vm_size"],
+                        arm_vm_password is not None, arm_vm_password)
+
     # create nfs server if specified.
     for i in range(int(config["azure_cluster"]["nfs_node_num"])):
             create_vm_param(i, "nfs", config["azure_cluster"]["nfs_vm_size"], False,
@@ -525,6 +530,8 @@ def create_vm_param(i, role, vm_size, no_az=False, arm_vm_password=None, vmcnf =
     elif role == "infra":
         vmname = "%s-infra%02d" % (config["azure_cluster"]
                                    ["cluster_name"], i + 1)
+    elif role == "mysql":
+        vmname = "%s-mysql%02d" % (config["azure_cluster"]["cluster_name"], i + 1)
     elif role == "dev":
         vmname = "%s-dev" % (config["azure_cluster"]["cluster_name"])
 
@@ -823,6 +830,15 @@ def gen_cluster_config(output_file_name, output_file=True, no_az=False):
                 cc["machines"][vmname.lower()] = {
                     "role": "worker",
                     "node-group": vm["vmSize"],"gpu-type":sku_mapping.get(vm["vmSize"],sku_mapping["default"])["gpu-type"]}
+
+    # Add mysql nodes
+    for vm in vm_list:
+        vmname = vm["name"]
+        if "-mysql" in vmname:
+            cc["machines"][vmname.lower()] = {
+                "role": "mysql",
+                "node-group": vm["vmSize"]}
+
     nfs_nodes = []
     for vm in vm_list:
         vmname = vm["name"]

src/ClusterBootstrap/deploy.py

@@ -58,7 +58,7 @@
 verbose = False
 nocache = False
 limitnodes = None
-allroles = {"infra", "infrastructure", "worker", "nfs", "sql", "samba"}
+allroles = {"infra", "infrastructure", "worker", "nfs", "sql", "samba", "mysql"}
 
 
 # default search for all partitions of hdb, hdc, hdd, and sdb, sdc, sdd
@@ -679,12 +679,14 @@ def check_master_ETCD_status():
     print "Checking Available Nodes for Deployment..."
     get_ETCD_master_nodes(config["clusterId"])
     get_worker_nodes(config["clusterId"], False)
+    get_nodes_by_roles(["mysql"])
     get_nodes_by_roles(["nfs"])
     get_nodes_by_roles(["samba"])
     print "==============================================="
     print "Activate Master Node(s): %s\n %s \n" % (len(config["kubernetes_master_node"]),",".join(config["kubernetes_master_node"]))
     print "Activate ETCD Node(s):%s\n %s \n" % (len(config["etcd_node"]),",".join(config["etcd_node"]))
     print "Activate Worker Node(s):%s\n %s \n" % (len(config["worker_node"]),",".join(config["worker_node"]))
+    print "Activate MySQL Node(s):%s\n %s \n" % (len(config["mysql_node"]), ",".join(config["mysql_node"]))
     print "Activate NFS Node(s):%s\n %s \n" % (len(config["nfs_node"]),",".join(config["nfs_node"]))
     print "Activate Samba Node(s):%s\n %s \n" % (len(config["samba_node"]), ",".join(config["samba_node"]))
 
@@ -1438,6 +1440,36 @@ def update_nfs_nodes(nargs):
     os.system("rm ./deploy/kubelet/worker-kubeconfig.yaml")
 
 
+def update_mysql_nodes(nargs):
+    """Internally use update_worker_node.
+
+    TODO: Should be covered by update_role_nodes in deploy.py V2
+    """
+    # This is to temporarily replace gpu_type with None to disallow nvidia runtime config to appear in /etc/docker/daemon.json
+    prev_gpu_type = config["gpu_type"]
+    config["gpu_type"] = "None"
+    utils.render_template_directory("./template/kubelet", "./deploy/kubelet", config)
+    config["gpu_type"] = prev_gpu_type
+
+    write_nodelist_yaml()
+
+    os.system('sed "s/##etcd_endpoints##/%s/" "./deploy/kubelet/options.env.template" > "./deploy/kubelet/options.env"' % config["etcd_endpoints"].replace("/", "\\/"))
+    os.system('sed "s/##api_servers##/%s/" ./deploy/kubelet/kubelet.service.template > ./deploy/kubelet/kubelet.service' % config["api_servers"].replace("/", "\\/"))
+    os.system('sed "s/##api_servers##/%s/" ./deploy/kubelet/worker-kubeconfig.yaml.template > ./deploy/kubelet/worker-kubeconfig.yaml' % config["api_servers"].replace("/", "\\/"))
+
+    get_hyperkube_docker()
+
+    mysql_nodes = get_nodes_by_roles(["mysql"])
+    mysql_nodes = limit_nodes(mysql_nodes)
+    for node in mysql_nodes:
+        if in_list(node, nargs):
+            update_worker_node(node)
+
+    os.system("rm ./deploy/kubelet/options.env")
+    os.system("rm ./deploy/kubelet/kubelet.service")
+    os.system("rm ./deploy/kubelet/worker-kubeconfig.yaml")
+
+
 def create_MYSQL_for_WebUI():
     #todo: create a mysql database, and set "mysql-hostname", "mysql-username", "mysql-password", "mysql-database"
     pass
@@ -2873,6 +2905,8 @@ def get_node_lists_for_service(service):
         nodetype = labels[service] if service in labels else labels["default"]
         if nodetype == "worker_node":
             nodes = config["worker_node"]
+        elif nodetype == "mysql_node":
+            nodes = config["mysql_node"]
         elif nodetype == "nfs_node":
             nodes = config["nfs_node"]
         elif nodetype == "etcd_node":
@@ -3307,7 +3341,7 @@ def run_command( args, command, nargs, parser ):
             role2connect = nargs[0]
             if len(nargs) < 1 or role2connect == "master":
                 nodes = config["kubernetes_master_node"]
-            elif role2connect in ["etcd", "worker", "nfs", "samba"]:
+            elif role2connect in ["etcd", "worker", "nfs", "samba", "mysql"]:
                 nodes = config["{}_node".format(role2connect)]
             else:
                 parser.print_help()
@@ -3498,6 +3532,13 @@ def run_command( args, command, nargs, parser ):
             gen_configs()
             reset_worker_nodes()
 
+    elif command == "updatemysql":
+        response = raw_input_with_default("Deploy MySQL Node(s) (y/n)?")
+        if first_char(response) == "y":
+            check_master_ETCD_status()
+            gen_configs()
+            update_mysql_nodes(nargs)
+
     elif command == "updatenfs":
         response = raw_input_with_default("Deploy NFS Node(s) (y/n)?")
         if first_char(response) == "y":

src/ClusterBootstrap/params.py

@@ -1,7 +1,7 @@
 # These are the default configuration parameter
 default_config_parameters = {
     "supported_platform": ["azure_cluster", "onpremise"],
-    "allroles": {"infra", "infrastructure", "worker", "nfs", "sql", "dev", "etcd", "kubernetes_master"},
+    "allroles": {"infra", "infrastructure", "worker", "nfs", "sql", "dev", "etcd", "kubernetes_master", "mysql"},
     # Kubernetes setting
     "service_cluster_ip_range": "10.3.0.0/16",
     "pod_ip_range": "10.2.0.0/16",
@@ -239,6 +239,7 @@
         "elasticsearch": "etcd_node_1",
         "kibana": "etcd_node_1",
         "mysql": "etcd_node_1",
+        "mysql-server": "mysql_node",
         "nginx": "all",
         "storagemanager": "nfs_node",
         "user-synchronizer": "etcd_node_1",

src/ClusterBootstrap/services/mysql/mysql.yaml

@@ -24,7 +24,11 @@ spec:
         app: mysql
     spec:
       nodeSelector:
+      {% if cnf["azure_cluster"]["mysql_node_num"]|int > 0 %}
+        mysql-server: active
+      {% else %}
         mysql: active
+      {% endif %}
       hostNetwork: true    
       containers:
       - image: {{cnf["dockers"]["container"]["mysql"]["fullname"]}}

src/ClusterBootstrap/services/mysql/phpmyadmin.yaml

@@ -4,8 +4,10 @@ metadata:
   name: phpmyadmin
   namespace: kube-system
 spec:
+  type: NodePort
   ports:
   - port: 80
+    targetPort: 80
   selector:
     app: phpmyadmin
 ---
@@ -24,7 +26,11 @@ spec:
         app: phpmyadmin
     spec:
       nodeSelector:
+      {% if cnf["azure_cluster"]["mysql_node_num"]|int > 0 %}
+        mysql-server: active
+      {% else %}
         mysql: active
+      {% endif %}
       containers:
       - image: {{cnf["dockers"]["container"]["phpmyadmin"]["fullname"]}}
         name: phpmyadmin
@@ -33,7 +39,7 @@ spec:
         - name: MYSQL_ROOT_PASSWORD
           value: "{{cnf["mysql_password"]}}"
         - name: PMA_HOST
-          value: {{cnf["mysql_node"]}}   
+          value: {{cnf["mysql_node"][0]}}
         - name: PMA_PORT
           value: "{{cnf["mysql_port"]}}"                   
         ports: