[PLAT-3207] Install yb-controller during universe creation behind enableYbc feature flag

Summary:
This commit adds enableYbc feature flag and installs yb-controller on DB nodes accordingly.
On enabling enableYbc feature flag, YBC package path field will appear on create universe page below DB version field.
This commit will download ybc package from location specified in YBC package path and try to install it on DB nodes.
If YBC package path field is left empty or it specifies a wrong path, then yb-controller will not be installed on nodes.

Note: For this to work, YBC package path should have same source as DB package. For example, if DB releases is in S3 then YBC package path should be a S3 URI.

{F24443}

{F24444}

Test Plan:
Manual.

Additional scenarios tested manually:
1. Add Read replica
2. Add node
3. Edit universe
4. Pause/resume/rolling restart

Steps:
Go to /features and enable the enableYbc feature flag.
Go to create universe, fill details and also pass Ybc package path.
Each node in universe should have yb-controller installed on it.

dev-itest-pipeline run: https://jenkins.dev.yugabyte.com/job/dev-itest-pipeline/810

Reviewers: vkumar, asathyan, lsangappa, cpadinjareveettil, kkg, sanketh, kkannan, vbansal, vpatibandla, amalyshev

Reviewed By: kkannan, vbansal, amalyshev

Subscribers: amalyshev, yshchetinin, nbhatia, jenkins-bot, yugaware

Differential Revision: https://phabricator.dev.yugabyte.com/D15953

Author: achauhan-yb

managed/build.sbt

@@ -379,6 +379,7 @@ runPlatform := {
 }
 
 libraryDependencies += "org.yb" % "yb-client" % "0.8.20-SNAPSHOT"
+libraryDependencies += "org.yb" % "ybc-client" % "0.0.1"
 
 libraryDependencies ++= Seq(
   // We wont use swagger-ui jar since we want to change some of the assets:

managed/devops/configure-cluster-server.yml

@@ -18,3 +18,6 @@
     - role: install_package
       symlink_folders: "{{ yb_process_type | default(['master', 'tserver']) }}"
       when: package is defined
+
+    - role: configure-ybc-server
+      when: configure_ybc

managed/devops/opscli/ybops/cloud/common/command.py

@@ -70,7 +70,8 @@ def __init__(self):
         super(ControlInstanceCommand, self).__init__("control")
         self.commands_per_server_type = {
             "master": self.MASTER_COMMANDS,
-            "tserver": self.BASE_COMMANDS
+            "tserver": self.BASE_COMMANDS,
+            "yb-controller": self.BASE_COMMANDS
             }
 
     def add_subcommands(self):

managed/devops/opscli/ybops/cloud/common/method.py

@@ -161,6 +161,10 @@ def add_extra_args(self):
                                  action="store_true",
                                  default=False,
                                  help="check if systemd services is set")
+        self.parser.add_argument("--configure_ybc",
+                                 action="store_true",
+                                 default=False,
+                                 help="configure yb-controller on node.")
         self.parser.add_argument("--machine_image",
                                  required=False,
                                  help="The machine image (e.g. an AMI on AWS) to install, "
@@ -690,6 +694,7 @@ def callback(self, args):
         self.extra_vars["use_chrony"] = args.use_chrony
         self.extra_vars.update({"systemd_option": args.systemd_services})
         self.extra_vars.update({"instance_type": args.instance_type})
+        self.extra_vars.update({"configure_ybc": args.configure_ybc})
         self.extra_vars["device_names"] = self.cloud.get_device_names(args)
 
         self.cloud.setup_ansible(args).run("yb-server-provision.yml", self.extra_vars, host_info)
@@ -947,11 +952,13 @@ def prepare(self):
         self.parser.add_argument('--package', default=None)
         self.parser.add_argument('--num_releases_to_keep', type=int,
                                  help="Number of releases to keep after upgrade.")
+        self.parser.add_argument('--ybc_package', default=None)
         self.parser.add_argument('--yb_process_type', default=None,
                                  choices=self.VALID_PROCESS_TYPES)
         self.parser.add_argument('--extra_gflags', default=None)
         self.parser.add_argument('--gflags', default=None)
         self.parser.add_argument('--gflags_to_remove', default=None)
+        self.parser.add_argument('--ybc_flags', default=None)
         self.parser.add_argument('--master_addresses_for_tserver')
         self.parser.add_argument('--master_addresses_for_master')
         self.parser.add_argument('--server_broadcast_addresses')
@@ -1037,6 +1044,7 @@ def callback(self, args):
                 self.supported_types))
 
         self.extra_vars["systemd_option"] = args.systemd_services
+        self.extra_vars["configure_ybc"] = args.configure_ybc
 
         # Make sure we set server_type so we pick the right configure.
         self.update_ansible_vars_with_args(args)
@@ -1051,6 +1059,11 @@ def callback(self, args):
 
         if args.num_releases_to_keep is not None:
             self.extra_vars["num_releases_to_keep"] = args.num_releases_to_keep
+        if args.ybc_package is not None:
+            self.extra_vars["ybc_package"] = args.ybc_package
+
+        if args.ybc_flags is not None:
+            self.extra_vars["ybc_flags"] = args.ybc_flags
 
         if args.extra_gflags is not None:
             self.extra_vars["extra_gflags"] = json.loads(args.extra_gflags)
@@ -1104,7 +1117,14 @@ def callback(self, args):
                         raise YBOpsRuntimeError("{} is not a valid s3 URI. Must match {}"
                                                 .format(args.package, s3_uri_pattern))
 
+                    if args.ybc_package is not None:
+                        match = re.match(s3_uri_pattern, args.ybc_package)
+                        if not match:
+                            raise YBOpsRuntimeError("{} is not a valid s3 URI. Must match {}"
+                                                    .format(args.ybc_package, s3_uri_pattern))
+
                     self.extra_vars['s3_package_path'] = args.package
+                    self.extra_vars['s3_ybc_package_path'] = args.ybc_package
                     self.extra_vars['aws_access_key'] = aws_access_key
                     self.extra_vars['aws_secret_key'] = aws_secret_key
                     logging.info(
@@ -1125,7 +1145,14 @@ def callback(self, args):
                         raise YBOpsRuntimeError("{} is not a valid gs URI. Must match {}"
                                                 .format(args.package, gcs_uri_pattern))
 
+                    if args.ybc_package is not None:
+                        match = re.match(gcs_uri_pattern, args.ybc_package)
+                        if not match:
+                            raise YBOpsRuntimeError("{} is not a valid gs URI. Must match {}"
+                                                    .format(args.ybc_package, gcs_uri_pattern))
+
                     self.extra_vars['gcs_package_path'] = args.package
+                    self.extra_vars['gcs_ybc_package_path'] = args.ybc_package
                     self.extra_vars['gcs_credentials_json'] = gcs_credentials_json
                     logging.info(
                         "Variables to download {} directly on the remote host added."
@@ -1139,13 +1166,19 @@ def callback(self, args):
 
                     # Remove query string part from http url.
                     self.extra_vars["package"] = match.group(1)
-
                     # Pass the complete http url to download the package.
                     self.extra_vars['http_package_path'] = match.group(0)
                     self.extra_vars['http_package_checksum'] = args.http_package_checksum
                     logging.info(
                         "Variables to download {} directly on the remote host added."
                         .format(args.package))
+
+                    match = re.match(http_url_pattern, args.ybc_package)
+                    if not match:
+                        raise YBOpsRuntimeError("{} is not a valid HTTP URL. Must match {}"
+                                                .format(args.ybc_package, http_url_pattern))
+                    self.extra_vars["http_ybc_package"] = match.group(0)
+
                 elif args.itest_s3_package_path and args.type == self.YB_SERVER_TYPE:
                     itest_extra_vars = self.extra_vars.copy()
                     itest_extra_vars["itest_s3_package_path"] = args.itest_s3_package_path
@@ -1168,6 +1201,19 @@ def callback(self, args):
                     logging.info("[app] Copying package {} to {} took {:.3f} sec".format(
                         args.package, args.search_pattern, time.time() - start_time))
 
+                    if args.ybc_package is not None:
+                        ybc_package_path = args.ybc_package
+                        if os.path.isfile(ybc_package_path):
+                            start_time = time.time()
+                            scp_to_tmp(
+                                ybc_package_path,
+                                self.extra_vars["private_ip"],
+                                self.extra_vars["ssh_user"],
+                                self.extra_vars["ssh_port"],
+                                args.private_key_file)
+                            logging.info("[app] Copying package {} to {} took {:.3f} sec".format(
+                                ybc_package_path, args.search_pattern, time.time() - start_time))
+
         logging.info("Configuring Instance: {}".format(args.search_pattern))
         ssh_options = {
             # TODO: replace with args.ssh_user when it's setup in the flow

managed/devops/roles/configure-cluster-server/templates/yb-server-ctl.sh.j2

@@ -31,6 +31,7 @@ Usage: ${0##*/} <daemon> <command> OR
 Daemons:
   master
   tserver
+  yb-controller
 Commands:
   create        - Start the YB process on this node in cluster creation node (only applicable for
                   master)
@@ -188,6 +189,12 @@ case "$daemon" in
       exit 1
     fi
   ;;
+  yb-controller)
+    if [ "$command" == "create" ]; then
+      echo "create command is not valid for yb-controller"
+      exit 1
+    fi
+  ;;
   *)
     echo "Invalid Daemon: $daemon"
     print_help
@@ -196,9 +203,16 @@ esac
 
 daemon_home="{{ yb_home_dir }}/$daemon"
 daemon_logs_dir="$daemon_home/logs"
-daemon_binary="$daemon_home/bin/yb-$daemon"
-daemon_conf="$daemon_home/conf/server.conf"
-daemon_pid_file="$daemon_logs_dir/yb-$daemon.pid"
+if [[ $daemon == yb-controller ]]; then
+  daemon_binary="$daemon_home/bin/$daemon-server"
+  daemon_conf="$daemon_home/conf/$daemon.conf"
+  daemon_pid_file="$daemon_logs_dir/$daemon.pid"
+else
+  daemon_binary="$daemon_home/bin/yb-$daemon"
+  daemon_conf="$daemon_home/conf/server.conf"
+  daemon_pid_file="$daemon_logs_dir/yb-$daemon.pid"
+fi
+
 
 case "$command" in
   create|start)

managed/devops/roles/configure-ybc-server/defaults/main.yml

@@ -0,0 +1,14 @@
+# Copyright 2019 YugaByte, Inc. and Contributors
+#
+# Licensed under the Polyform Free Trial License 1.0.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# https://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt
+
+yb_home_dir: "/home/{{ user_name }}"
+yb_bin_dir: "{{ yb_home_dir }}/bin"
+ybc_conf_file: "{{ yb_home_dir }}/yb-controller/conf/yb-controller.conf"
+ybc_logs_dir: "{{ yb_home_dir }}/yb-controller/logs"
+configure_ybc: false
+systemd_option: false

managed/devops/roles/configure-ybc-server/tasks/main.yml

@@ -0,0 +1,112 @@
+- assert:
+    that:
+      - user_name is defined
+      - systemd_option is defined
+
+- block:
+  - name: Setup | Create yb-controller logs directory
+    file:
+      path: "{{ ybc_logs_dir }}"
+      state: directory
+      mode: 0755
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+
+  - name: Remove existing ybc conf file
+    file:
+      path: "{{ ybc_conf_file }}"
+      state: absent
+    tags:
+    - override_gflags
+    when: ybc_flags is defined
+
+  - name: Configure | Set ybc flags
+    lineinfile:
+      dest: "{{ ybc_conf_file }}"
+      regexp: "^--{{ item.key }}=(.*)"
+      line: "--{{item.key}}={{item.value}}"
+      create: yes
+    with_dict: "{{ ybc_flags }}"
+    tags:
+    - override_gflags
+    when: ybc_flags is defined
+
+- block:
+  # Cronjob approach for starting yb-controller
+  - name: Setup cron to check liveness of yb-controller
+    cron:
+      name: "Check liveness of yb-controller"
+      minute: "*/1"
+      user: "{{ user_name }}"
+      job: >
+        {{ yb_bin_dir }}/yb-server-ctl.sh yb-controller cron-check ||
+        {{ yb_bin_dir }}/yb-server-ctl.sh yb-controller start
+    register: cron_result
+    failed_when:
+      - cron_result is failed
+      - cloud_type != "onprem"
+    when: not systemd_option
+    tags: yb-prebuilt-ami
+
+- block:
+  # Remove old cron job and its tasks that start yb-controller
+  - name: Remove old cron job that starts yb-controller (for upgrades)
+    cron:
+      name: "Check liveness of yb-controller"
+      user: "yugabyte"
+      state: absent
+
+  - name: User mode systemd
+    block:
+    - name: Stop existing running yb-controller
+      systemd:
+        name: yb-controller
+        state: stopped
+        scope: user
+
+    - name: Reload systemd user daemon
+      systemd:
+        daemon_reload: yes
+        scope: user
+
+    # User units need to be linked if waiting on system units
+    - name: Link network-online.target
+      shell:
+        cmd: "systemctl --user link /lib/systemd/system/network-online.target"
+
+    - name: Start network-online.target
+      systemd:
+        name: network-online.target
+        state: started
+        enabled: yes
+        scope: user
+
+    - name: Enabling and starting yb-controller
+      systemd:
+        name: yb-controller
+        state: started
+        enabled: yes
+        scope: user
+
+  - name: System (RHEL7) systemd units
+    block:
+    - name: Stop existing running yb-controller
+      shell:
+        cmd: "sudo systemctl stop yb-controller"
+
+    # Systemd daemon-reload for yb-controller service files
+    - name: Perform daemon-reload for the new services
+      shell:
+        cmd: "sudo systemctl daemon-reload"
+
+    - name: Enable yb-controller service
+      shell:
+        cmd: "sudo systemctl enable yb-controller"
+
+    - name: Run yb-controller.service to start the yb-controller
+      shell:
+        cmd: "sudo systemctl start yb-controller"
+  tags:
+    - override_gflags
+    - systemd_upgrade
+  when: systemd_option

managed/devops/roles/install_package/defaults/main.yml

@@ -11,3 +11,4 @@ yb_software_dir: "{{ yb_home_dir }}/yb-software"
 itest_s3_package_path: ""
 num_retries: 5
 retry_delay: 10
+configure_ybc: false