azure.cdsw.conf

#
# Copyright (c) 2018-2019 Cloudera, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Sample Cloudera Altus Director configuration file based on the Cloudera Azure reference architecture:
# http://www.cloudera.com/documentation/other/reference-architecture/PDF/cloudera_ref_arch_azure.pdf
#
# This is a template for a minimal deployment of CDSW. Refer to Cloudera Altus Director's documentation
# and azure.reference.conf for more information about deploying clusters with Cloudera Altus Director.
#

#
# Cluster name
#

name: cdsw

#
# Cloud provider configuration (credentials, region or zone, and more)
#

provider {
    type: azure

    #
    # ID of Azure region to use. NOTE: region must support Premium Storage
    # See: https://azure.microsoft.com/en-us/global-infrastructure/regions/
    #

    region: "region_REPLACE_ME"

    #
    # Azure Cloud Environment to use. Valid values are:
    #   - azure
    #   - azure-us-government
    #   - azure-germany
    #

    azureCloudEnvironment: "azureCloudEnvironment_REPLACE_ME"

    #
    # Azure Active Directory Subscription ID.
    #

    subscriptionId: "subscriptionId_REPLACE_ME"

    #
    # Tenant ID (from AAD)
    #

    tenantId: "tenantId_REPLACE_ME"

    #
    # Azure Active Directory Application Client ID.
    #

    clientId: "clientId_REPLACE_ME"

    #
    # Client Secret
    #

    clientSecret: "clientSecret_REPLACE_ME"
}

#
# SSH credentials to use to connect to the machines
#

ssh {
    username: "username_REPLACE_ME"
    privateKey: privateKey_REPLACE_ME # with an absolute path to .pem file, ${HOME} may be used
}

#
# Instance templates
#
# See azure.reference.conf for a breakdown of instance template fields.
#

instances {

    # fields that are common to all nodes
    base {
        type: STANDARD_D32S_V3
        image: cloudera-centos-74-latest
        networkSecurityGroupResourceGroup: "networkSecurityGroupResourceGroup_REPLACE_ME"
        networkSecurityGroup: "networkSecurityGroup_REPLACE_ME"
        virtualNetworkResourceGroup: "virtualNetworkResourceGroup_REPLACE_ME"
        virtualNetwork: "virtualNetwork_REPLACE_ME"
        subnetName: "subnetName_REPLACE_ME"
        hostFqdnSuffix: "hostFqdnSuffix_REPLACE_ME"
        tags {
            owner: ${?USER}
        }
        bootstrapScripts: [ ${bootstrap-script.os-generic}, ${bootstrap-script.forcejdk8}]
    }

    # fields for master nodes
    master: ${instances.base} {
        computeResourceGroup: "master_computeResourceGroup_REPLACE_ME"
        availabilitySet: "master_availabilitySet_REPLACE_ME"
        instanceNamePrefix: "master_instanceNamePrefix_REPLACE_ME"
        storageAccountType: "Premium_LRS"
        dataDiskCount: 3
        dataDiskSize: 512
        managedDisks: Yes
        publicIP: No
    }

    # fields for worker nodes
    worker: ${instances.base} {
        computeResourceGroup: "worker_computeResourceGroup_REPLACE_ME"
        availabilitySet: "worker_availabilitySet_REPLACE_ME"
        instanceNamePrefix: "worker_instanceNamePrefix_REPLACE_ME"
        storageAccountType: "Premium_LRS"
        dataDiskCount: 4
        dataDiskSize: 1024
        managedDisks: Yes
        publicIP: No
    }

    # fields for edge nodes
    edge: ${instances.base} {
        computeResourceGroup: "edge_computeResourceGroup_REPLACE_ME"
        availabilitySet: "edge_availabilitySet_REPLACE_ME"
        instanceNamePrefix: "edge_instanceNamePrefix_REPLACE_ME"
        storageAccountType: "Premium_LRS"
        dataDiskCount: 1
        dataDiskSize: 512
        managedDisks: Yes
        # Change this to Yes to allow accessing edge/CM nodes via public IP
        publicIP: No
    }

    # fields for CDSW master nodes
    cdswmaster: ${instances.base} {
        computeResourceGroup: "cdswmaster_computeResourceGroup_REPLACE_ME"
        availabilitySet: "cdswmaster_availabilitySet_REPLACE_ME"
        instanceNamePrefix: "cdswmaster_instanceNamePrefix_REPLACE_ME"
         # Use an empty subnet dedicated to the cdsw master node to get predictable IP
         # that can be registered in DNS. Example, if subnet is 10.6.1.0/29 (3
         # available addresses in Azure) the master node will be assigned 10.6.1.4
         # See http://tiny.cloudera.com/azure-available-ip for details.
        subnetName: "masterSubnetName_REPLACE_ME"
        storageAccountType: "Premium_LRS"
        dataDiskCount: 3
        dataDiskSize: 1024
        normalizationConfig {
            mountAllUnmountedDisks: false
        }
        managedDisks: Yes
        publicIP: No
        bootstrapScripts: [ ${bootstrap-script.os-generic}, ${bootstrap-script.forcejdk8}, ${bootstrap-script.setupcdswmount}]
    }

    # fields for CDSW worker nodes
    cdswworker: ${instances.base} {
        computeResourceGroup: "cdswworker_computeResourceGroup_REPLACE_ME"
        availabilitySet: "cdswworker_availabilitySet_REPLACE_ME"
        instanceNamePrefix: "cdswworker_instanceNamePrefix_REPLACE_ME"
        storageAccountType: "Premium_LRS"
        dataDiskCount: 2
        dataDiskSize: 1024
        normalizationConfig {
            mountAllUnmountedDisks: false
        }
        managedDisks: Yes
        publicIP: No
    }
}

#
# Bootstrap scripts
#
# See azure.reference.conf for general information about bootstrap scripts and
# pre-terminate scripts.
#
# This configuration opts to install an Oracle JDK via bootstrap script.

include "azure-os-generic-bootstrap.conf"

bootstrap-script {
    forcejdk8: """#!/bin/sh

# Remove any natively installed JDKs
yum remove --assumeyes *openjdk*

# Install JDK 1.8 required by Spark 2
rpm -ivh "https://archive.cloudera.com/director6/6.0.0/redhat7/RPMS/x86_64/oracle-j2sdk1.8-1.8.0+update141-1.x86_64.rpm"
"""
    setupcdswmount: """#!/bin/sh
set -e

# Mount one volume for application data
device="/dev/disk/azure/scsi1/lun2"
mount="/var/lib/cdsw"

echo "Making file system"
mkfs.ext4 -F -E lazy_itable_init=1 "$device" -m 0

echo "Mounting $device on $mount"
if [ ! -e "$mount" ]; then
  mkdir -p "$mount"
fi

mount -o defaults,noatime "$device" "$mount"
echo "$device $mount ext4 defaults,noatime 0 0" >> /etc/fstab

exit 0
"""
}

#
# Configuration for Cloudera Manager. Cloudera Altus Director can use an existing Cloudera
# Manager installation, or bootstrap everything from scratch for a new cluster.
#

cloudera-manager {

    instance: ${instances.edge} {

        #
        # Additional tags for the Cloudera Manager instance
        #

        tags {
            application: "Cloudera Manager 5"
        }
    }

    #
    # Select the strategy for installing a JDK. Choices are:
    #
    # - AUTO (default): Altus Director installs the JDK on the Cloudera Manager instance, and
    #   Cloudera Manager installs the JDK on cluster instances
    # - DIRECTOR_MANAGED: Altus Director installs the JDK on all instances
    # - NONE: Neither Altus Director nor Cloudera Manager installs the JDK on any instances
    #
    # JDK 1.8 is installed with a bootstrap script, so the NONE strategy is selected
    #

    javaInstallationStrategy: NONE

    #
    # Configuration to override Cloudera Manager package repositories. These are
    # optional, and default to the Cloudera Enterprise release corresponding to
    # the Altus Director version.
    #
    # Support for CDSW requires Cloudera Manager 5.13.1+
    #

    repository: "https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.15/"
    repositoryKeyUrl: "https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera"

    #
    # Configuration for Cloudera Manager and its management services
    #

    configs {

        #
        # CLOUDERA_MANAGER corresponds to the Cloudera Manager Server configuration options
        #

        CLOUDERA_MANAGER {
            # enable_api_debug: true
            custom_banner_html: "Managed by Cloudera Altus Director"
        }

        SERVICEMONITOR {
            mgmt_log_dir: /data0/log/cloudera-scm-firehose
            firehose_storage_dir: /data0/lib/cloudera-service-monitor
        }

        ACTIVITYMONITOR {
            mgmt_log_dir: /data0/log/cloudera-scm-firehose
        }

        HOSTMONITOR {
            mgmt_log_dir: /data0/log/cloudera-scm-firehose
            firehose_storage_dir: /data0/lib/cloudera-host-monitor
        }

        REPORTSMANAGER {
            headlamp_scratch_dir: /data0/lib/cloudera-scm-headlamp
            mgmt_log_dir: /data0/log/cloudera-scm-headlamp
        }

        EVENTSERVER {
            mgmt_log_dir: /data0/log/cloudera-scm-eventserver
            eventserver_index_dir: /data0/lib/cloudera-scm-eventserver
        }

        ALERTPUBLISHER {
            mgmt_log_dir: /data0/log/cloudera-scm-alertpublisher
        }

        NAVIGATOR {
            mgmt_log_dir: /data0/log/cloudera-scm-navigator
        }

        NAVIGATORMETASERVER {
            audit_event_log_dir: /data0/log/cloudera-scm-navigator/audit
            data_dir: /data0/lib/cloudera-scm-navigator
            mgmt_log_dir: /data0/log/cloudera-scm-navigator
        }

    }

    #
    # Optional custom service descriptors for external parcels
    #
    # Custom service descriptors for CDSW and Spark2
    #

    csds: [
        "https://archive.cloudera.com/cdsw1/1.4.2/csd/CLOUDERA_DATA_SCIENCE_WORKBENCH-CDH5-1.4.2.jar",
        "https://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.3.0.cloudera4.jar"
    ]
}

#
# Cluster description
#

cluster {

    # The table of products and their versions that need to be installed. Each
    # product must have a corresponding parcel in the parcelRepositories
    # configured in this section. The specified version for a product will be
    # used to find a suitable parcel. Specifying a version that is satisfied by
    # more than one parcel among those available will result in a configuration
    # error. Specify more granular versions to avoid conflicts.

    products {
      CDH: 5.15
      CDSW: 1.4.2
      SPARK2: 2
    }

    #
    # Optional override of CDH parcel repositories
    #
    # This defaults to the Cloudera Enterprise release corresponding to the
    # Altus Director version.
    #

    parcelRepositories: [ "https://archive.cloudera.com/cdh5/parcels/5.15/",
                         "https://archive.cloudera.com/cdsw1/1.4.2/parcels/",
                         "https://archive.cloudera.com/spark2/parcels/2.3.0.cloudera4/"]

    #
    # Services to include in the cluster
    #

    services: [
        HDFS,
        YARN,
        ZOOKEEPER,
        CDSW,
        SPARK2_ON_YARN
    ]

    #
    # Custom service configurations
    #

    configs {

        CDSW {
            "cdsw.domain.config": cdsw.my-domain.com # The fully qualified domain name for the CDSW host
        }

    }

    #
    # Instance group configurations
    #

    masters {
        count: 1
        instance: ${instances.master} {
            tags: {
                application: "CDH 5.15"
                group: masters
            }
        }

        roles {
            HDFS: [NAMENODE, SECONDARYNAMENODE]
            YARN: [RESOURCEMANAGER, JOBHISTORY]
            SPARK2_ON_YARN: [SPARK2_YARN_HISTORY_SERVER]
        }

        configs {

            HDFS {
                NAMENODE {
                    namenode_log_dir: /data0/log/hadoop-hdfs
                    dfs_name_dir_list: /data1/dfs/nn
                }
                SECONDARYNAMENODE {
                    secondarynamenode_log_dir: /data0/log/hadoop-hdfs
                    fs_checkpoint_dir_list: /data2/dfs/snn
                }
            }

            YARN {
                RESOURCEMANAGER {
                    resource_manager_log_dir: /data0/log/hadoop-yarn
                }
                JOBHISTORY {
                    mr2_jobhistory_log_dir: /data0/log/hadoop-mapreduce
                }
            }

            SPARK_ON_YARN {
                SPARK_YARN_HISTORY_SERVER {
                    log_dir: /data0/log/spark2
                }
            }

        }
    }

    workers {
        count: 3
        minCount: 3
        instance: ${instances.worker} {
            tags: {
                application: "CDH 5.15"
                group: workers
            }
        }
        roles {
            HDFS: [DATANODE]
            YARN: [NODEMANAGER]
            ZOOKEEPER: [SERVER]
        }

        configs {

            HDFS {
                DATANODE {
                    datanode_log_dir: /data0/log/hadoop-hdfs
                    dfs_data_dir_list: "/data2/dfs/dn,/data3/dfs/dn"
                }
            }

            YARN {
                NODEMANAGER {
                    node_manager_log_dir: /data0/log/hadoop-yarn
                    yarn_nodemanager_log_dirs: "/data2/log/hadoop-yarn/container,/data3/log/hadoop-yarn/container"
                    yarn_nodemanager_local_dirs: "/data2/yarn,/data3/yarn"
                }
            }

            ZOOKEEPER {
                SERVER {
                    zk_server_log_dir: /data0/log/zookeeper
                    dataDir: /data1/zookeeper
                    dataLogDir: /data1/zookeeper
                }
            }

        }
    }

    cdswmasters {
        count: 1
        instance: ${instances.cdswmaster} {
            tags: {
                application: "CDH 5.15 + CDSW 1.4.2"
                group: cdswmasters
            }
        }

        roles {
            HDFS: [GATEWAY]
            YARN: [GATEWAY]
            CDSW: [CDSW_MASTER, CDSW_APPLICATION, CDSW_DOCKER]
            SPARK2_ON_YARN: [GATEWAY]
        }

        configs {
            CDSW {
                CDSW_DOCKER {
                    "cdsw.docker.devices.config": "/dev/disk/azure/scsi1/lun0 /dev/disk/azure/scsi1/lun1" # related to the data disk configuration
                }
            }
        }
    }

    cdswworkers {
        count: 2
        minCount: 2
        instance: ${instances.cdswworker} {
            tags: {
                application: "CDH 5.15 + CDSW 1.4.2"
                group: cdswworkers
            }
        }

        roles {
            HDFS: [GATEWAY]
            YARN: [GATEWAY]
            CDSW: [CDSW_WORKER, CDSW_DOCKER]
            SPARK2_ON_YARN: [GATEWAY]
        }

        configs {
            CDSW {
                CDSW_DOCKER {
                    "cdsw.docker.devices.config": "/dev/disk/azure/scsi1/lun0 /dev/disk/azure/scsi1/lun1" # related to the data disk configuration
                }
            }
        }
    }
}